def fill_ranges(self, start, end, length): """ Fill the request ranges. """ if length == 0: return if self.align and self.buf_size: # discard bytes # so we only yield complete EC segments self.discard_bytes = discard_bytes(self.buf_size, start) # change headers for efficient recovery if 'Range' in self.request_headers: try: orig_ranges = ranges_from_http_header( self.request_headers['Range']) new_ranges = [(start, end)] + orig_ranges[1:] except ValueError: new_ranges = [(start, end)] else: new_ranges = [(start, end)] self.request_headers['Range'] = http_header_from_ranges( new_ranges)
def get_object_fetch_resp(self, req): storage = self.app.storage if req.headers.get('Range'): ranges = ranges_from_http_header(req.headers.get('Range')) else: ranges = None oio_headers = {REQID_HEADER: self.trans_id} force_master = False while True: try: metadata, stream = storage.object_fetch( self.account_name, self.container_name, self.object_name, ranges=ranges, headers=oio_headers, version=req.environ.get('oio.query', {}).get('version'), force_master=force_master) break except (exceptions.NoSuchObject, exceptions.NoSuchContainer): if force_master \ or not self.container_name.endswith('+segments'): # Either the request failed with the master, # or it is not an MPU return HTTPNotFound(request=req) # This part appears in the manifest, so it should be there. # To be sure, we must go check the master # in case of desynchronization. force_master = True resp = self.make_object_response(req, metadata, stream) return resp
def _convert_range(self, req_start, req_end, length): try: ranges = ranges_from_http_header("bytes=%s-%s" % ( req_start if req_start is not None else '', req_end if req_end is not None else '')) except ValueError: return (None, None) result = fix_ranges(ranges, length) if not result: return (None, None) else: return (result[0][0], result[0][1])
def get_object_fetch_resp(self, req): storage = self.app.storage if req.headers.get('Range'): ranges = ranges_from_http_header(req.headers.get('Range')) else: ranges = None oio_headers = {'X-oio-req-id': self.trans_id} try: metadata, stream = storage.object_fetch( self.account_name, self.container_name, self.object_name, ranges=ranges, headers=oio_headers, version=req.environ.get('oio.query', {}).get('version')) except (exceptions.NoSuchObject, exceptions.NoSuchContainer): return HTTPNotFound(request=req) resp = self.make_object_response(req, metadata, stream, ranges=ranges) return resp
def get_object_fetch_resp(self, req): storage = self.app.storage if req.headers.get('Range'): ranges = ranges_from_http_header(req.headers.get('Range')) else: ranges = None try: metadata, stream = storage.object_fetch(self.account_name, self.container_name, self.object_name, ranges=ranges, version=req.environ.get( 'oio_query', {}).get('version')) except (exceptions.NoSuchObject, exceptions.NoSuchContainer): return HTTPNotFound(request=req) resp = self.make_object_response(req, metadata, stream, ranges=ranges) return resp
def recover(self, nb_bytes): """ Recover the request. :param nb_bytes: number of bytes already consumed that we need to discard if we perform a recovery from another source. :raises `ValueError`: if range header is not valid :raises `oio.common.exceptions.UnsatisfiableRange`: :raises `oio.common.exceptions.EmptyByteRange`: """ if 'Range' in self.request_headers: request_range = ranges_from_http_header( self.request_headers['Range']) start, end = request_range[0] if start is None: # suffix byte range end -= nb_bytes else: start += nb_bytes if end is not None: if start == end + 1: # no more bytes to serve in the requested byte range raise exc.EmptyByteRange() if start > end: # invalid range raise exc.UnsatisfiableRange() if end and start: # full byte range request_range = [(start, end)] + request_range[1:] else: # suffix byte range request_range = [(None, end)] + request_range[1:] else: # prefix byte range request_range = [(start, None)] + request_range[1:] self.request_headers['Range'] = http_header_from_ranges( request_range) else: # just add an offset to the request self.request_headers['Range'] = 'bytes=%d-' % nb_bytes
def recover(self, nb_bytes): """ Recover the request. :params nb_bytes: number of bytes already consumed that we need to discard if we perform a recovery from another source. :raises ValueError: if range header is not valid :raises UnsatisfiableRange :raises EmptyByteRange """ if "Range" in self.request_headers: request_range = ranges_from_http_header(self.request_headers["Range"]) start, end = request_range[0] if start is None: # suffix byte range end -= nb_bytes else: start += nb_bytes if end is not None: if start == end + 1: # no more bytes to serve in the requested byte range raise exc.EmptyByteRange() if start > end: # invalid range raise exc.UnsatisfiableRange() if end and start: # full byte range request_range = [(start, end)] + request_range[1:] else: # suffix byte range request_range = [(None, end)] + request_range[1:] else: # prefix byte range request_range = [(start, None)] + request_range[1:] self.request_headers["Range"] = http_header_from_ranges(request_range) else: # just add an offset to the request self.request_headers["Range"] = "bytes=%d-" % nb_bytes
def fill_ranges(self, start, end, length): """ Fill the request ranges. """ if length == 0: return if self.buf_size: # discard bytes # so we only yield complete EC segments self.discard_bytes = discard_bytes(self.buf_size, start) # change headers for efficient recovery if "Range" in self.request_headers: try: orig_ranges = ranges_from_http_header(self.request_headers["Range"]) new_ranges = [(start, end)] + orig_ranges[1:] except ValueError: new_ranges = [(start, end)] else: new_ranges = [(start, end)] self.request_headers["Range"] = http_header_from_ranges(new_ranges)
def _link_object(self, req): _, container, obj = req.headers['Oio-Copy-From'].split('/', 2) from_account = req.headers.get('X-Copy-From-Account', self.account_name) self.app.logger.info( "Creating link from %s/%s/%s to %s/%s/%s", # Existing from_account, container, obj, # New self.account_name, self.container_name, self.object_name) storage = self.app.storage if req.headers.get('Range'): raise Exception("Fast Copy with Range is unsupported") ranges = ranges_from_http_header(req.headers.get('Range')) if len(ranges) != 1: raise HTTPInternalServerError( request=req, body="mutiple ranges unsupported") ranges = ranges[0] else: ranges = None headers = self._prepare_headers(req) metadata = self.load_object_metadata(headers) oio_headers = {REQID_HEADER: self.trans_id} oio_cache = req.environ.get('oio.cache') perfdata = req.environ.get('swift.perfdata') # FIXME(FVE): use object_show, cache in req.environ version = obj_version_from_env(req.environ) props = storage.object_get_properties(from_account, container, obj, headers=oio_headers, version=version, cache=oio_cache, perfdata=perfdata) if props['properties'].get(SLO, None): raise Exception("Fast Copy with SLO is unsupported") else: if ranges: raise HTTPInternalServerError( request=req, body="no range supported with single object") try: # TODO check return code (values ?) link_meta = storage.object_link(from_account, container, obj, self.account_name, self.container_name, self.object_name, headers=oio_headers, properties=metadata, properties_directive='REPLACE', target_version=version, cache=oio_cache, perfdata=perfdata) # TODO(FVE): this exception catching block has to be refactored # TODO check which ones are ok or make non sense except exceptions.Conflict: raise HTTPConflict(request=req) except exceptions.PreconditionFailed: raise HTTPPreconditionFailed(request=req) except exceptions.SourceReadError: req.client_disconnect = True self.app.logger.warning( _('Client disconnected without sending last chunk')) self.app.logger.increment('client_disconnects') raise HTTPClientDisconnect(request=req) except exceptions.EtagMismatch: return HTTPUnprocessableEntity(request=req) except (exceptions.ServiceBusy, exceptions.OioTimeout, exceptions.DeadlineReached): raise except (exceptions.NoSuchContainer, exceptions.NotFound): raise HTTPNotFound(request=req) except exceptions.ClientException as err: # 481 = CODE_POLICY_NOT_SATISFIABLE if err.status == 481: raise exceptions.ServiceBusy() self.app.logger.exception( _('ERROR Exception transferring data %s'), {'path': req.path}) raise HTTPInternalServerError(request=req) except Exception: self.app.logger.exception( _('ERROR Exception transferring data %s'), {'path': req.path}) raise HTTPInternalServerError(request=req) resp = HTTPCreated(request=req, etag=link_meta['hash']) return resp
def _link_object(self, req): _, container, obj = req.headers['Oio-Copy-From'].split('/', 2) from_account = req.headers.get('X-Copy-From-Account', self.account_name) self.app.logger.info("LINK (%s,%s,%s) TO (%s,%s,%s)", from_account, self.container_name, self.object_name, self.account_name, container, obj) storage = self.app.storage if req.headers.get('Range'): raise Exception("Fast Copy with Range is unsupported") ranges = ranges_from_http_header(req.headers.get('Range')) if len(ranges) != 1: raise HTTPInternalServerError( request=req, body="mutiple ranges unsupported") ranges = ranges[0] else: ranges = None headers = self._prepare_headers(req) metadata = self.load_object_metadata(headers) oio_headers = {'X-oio-req-id': self.trans_id} # FIXME(FVE): use object_show, cache in req.environ props = storage.object_get_properties(from_account, container, obj) if props['properties'].get(SLO, None): raise Exception("Fast Copy with SLO is unsupported") if ranges is None: raise HTTPInternalServerError(request=req, body="LINK a MPU requires range") self.app.logger.debug("LINK, original object is a SLO") # retrieve manifest _, data = storage.object_fetch(from_account, container, obj) manifest = json.loads("".join(data)) offset = 0 # identify segment to copy for entry in manifest: if (ranges[0] == offset and ranges[1] + 1 == offset + entry['bytes']): _, container, obj = entry['name'].split('/', 2) checksum = entry['hash'] self.app.logger.info( "LINK SLO (%s,%s,%s) TO (%s,%s,%s)", from_account, self.container_name, self.object_name, self.account_name, container, obj) break offset += entry['bytes'] else: raise HTTPInternalServerError( request=req, body="no segment matching range") else: checksum = props['hash'] if ranges: raise HTTPInternalServerError( request=req, body="no range supported with single object") try: # TODO check return code (values ?) storage.object_fastcopy( from_account, container, obj, self.account_name, self.container_name, self.object_name, headers=oio_headers, properties=metadata, properties_directive='REPLACE') # TODO(FVE): this exception catching block has to be refactored # TODO check which ones are ok or make non sense except exceptions.Conflict: raise HTTPConflict(request=req) except exceptions.PreconditionFailed: raise HTTPPreconditionFailed(request=req) except exceptions.SourceReadError: req.client_disconnect = True self.app.logger.warning( _('Client disconnected without sending last chunk')) self.app.logger.increment('client_disconnects') raise HTTPClientDisconnect(request=req) except exceptions.EtagMismatch: return HTTPUnprocessableEntity(request=req) except (exceptions.ServiceBusy, exceptions.OioTimeout): raise except (exceptions.NoSuchContainer, exceptions.NotFound): raise HTTPNotFound(request=req) except exceptions.ClientException as err: # 481 = CODE_POLICY_NOT_SATISFIABLE if err.status == 481: raise exceptions.ServiceBusy() self.app.logger.exception( _('ERROR Exception transferring data %s'), {'path': req.path}) raise HTTPInternalServerError(request=req) except Exception: self.app.logger.exception( _('ERROR Exception transferring data %s'), {'path': req.path}) raise HTTPInternalServerError(request=req) resp = HTTPCreated(request=req, etag=checksum) return resp
def _link_object(self, req): _, container, obj = req.headers['Oio-Copy-From'].split('/', 2) from_account = req.headers.get('X-Copy-From-Account', self.account_name) self.app.logger.info("LINK (%s,%s,%s) TO (%s,%s,%s)", from_account, self.container_name, self.object_name, self.account_name, container, obj) storage = self.app.storage if req.headers.get('Range'): raise Exception("Fast Copy with Range is unsupported") ranges = ranges_from_http_header(req.headers.get('Range')) if len(ranges) != 1: raise HTTPInternalServerError( request=req, body="mutiple ranges unsupported") ranges = ranges[0] else: ranges = None headers = self._prepare_headers(req) metadata = self.load_object_metadata(headers) oio_headers = {'X-oio-req-id': self.trans_id} # FIXME(FVE): use object_show, cache in req.environ version = req.environ.get('oio.query', {}).get('version') props = storage.object_get_properties(from_account, container, obj, headers=oio_headers, version=version) if props['properties'].get(SLO, None): raise Exception("Fast Copy with SLO is unsupported") else: if ranges: raise HTTPInternalServerError( request=req, body="no range supported with single object") try: # TODO check return code (values ?) link_meta = storage.object_link( from_account, container, obj, self.account_name, self.container_name, self.object_name, headers=oio_headers, properties=metadata, properties_directive='REPLACE', target_version=version) # TODO(FVE): this exception catching block has to be refactored # TODO check which ones are ok or make non sense except exceptions.Conflict: raise HTTPConflict(request=req) except exceptions.PreconditionFailed: raise HTTPPreconditionFailed(request=req) except exceptions.SourceReadError: req.client_disconnect = True self.app.logger.warning( _('Client disconnected without sending last chunk')) self.app.logger.increment('client_disconnects') raise HTTPClientDisconnect(request=req) except exceptions.EtagMismatch: return HTTPUnprocessableEntity(request=req) except (exceptions.ServiceBusy, exceptions.OioTimeout): raise except (exceptions.NoSuchContainer, exceptions.NotFound): raise HTTPNotFound(request=req) except exceptions.ClientException as err: # 481 = CODE_POLICY_NOT_SATISFIABLE if err.status == 481: raise exceptions.ServiceBusy() self.app.logger.exception( _('ERROR Exception transferring data %s'), {'path': req.path}) raise HTTPInternalServerError(request=req) except Exception: self.app.logger.exception( _('ERROR Exception transferring data %s'), {'path': req.path}) raise HTTPInternalServerError(request=req) resp = HTTPCreated(request=req, etag=link_meta['hash']) return resp
def _link_object(self, req): _, container, obj = req.headers['Oio-Copy-From'].split('/', 2) from_account = req.headers.get('X-Copy-From-Account', self.account_name) self.app.logger.info("LINK (%s,%s,%s) TO (%s,%s,%s)", from_account, self.container_name, self.object_name, self.account_name, container, obj) storage = self.app.storage if req.headers.get('Range'): raise Exception("Fast Copy with Range is unsupported") ranges = ranges_from_http_header(req.headers.get('Range')) if len(ranges) != 1: raise HTTPInternalServerError( request=req, body="mutiple ranges unsupported") ranges = ranges[0] else: ranges = None headers = self._prepare_headers(req) metadata = self.load_object_metadata(headers) oio_headers = {'X-oio-req-id': self.trans_id} # FIXME(FVE): use object_show, cache in req.environ props = storage.object_get_properties(from_account, container, obj) if props['properties'].get(SLO, None): raise Exception("Fast Copy with SLO is unsupported") if ranges is None: raise HTTPInternalServerError(request=req, body="LINK a MPU requires range") self.app.logger.debug("LINK, original object is a SLO") # retrieve manifest _, data = storage.object_fetch(from_account, container, obj) manifest = json.loads("".join(data)) offset = 0 # identify segment to copy for entry in manifest: if (ranges[0] == offset and ranges[1] + 1 == offset + entry['bytes']): _, container, obj = entry['name'].split('/', 2) checksum = entry['hash'] self.app.logger.info("LINK SLO (%s,%s,%s) TO (%s,%s,%s)", from_account, self.container_name, self.object_name, self.account_name, container, obj) break offset += entry['bytes'] else: raise HTTPInternalServerError(request=req, body="no segment matching range") else: checksum = props['hash'] if ranges: raise HTTPInternalServerError( request=req, body="no range supported with single object") try: # TODO check return code (values ?) storage.object_fastcopy(from_account, container, obj, self.account_name, self.container_name, self.object_name, headers=oio_headers, properties=metadata, properties_directive='REPLACE') # TODO(FVE): this exception catching block has to be refactored # TODO check which ones are ok or make non sense except exceptions.Conflict: raise HTTPConflict(request=req) except exceptions.PreconditionFailed: raise HTTPPreconditionFailed(request=req) except exceptions.SourceReadError: req.client_disconnect = True self.app.logger.warning( _('Client disconnected without sending last chunk')) self.app.logger.increment('client_disconnects') raise HTTPClientDisconnect(request=req) except exceptions.EtagMismatch: return HTTPUnprocessableEntity(request=req) except (exceptions.ServiceBusy, exceptions.OioTimeout): raise except (exceptions.NoSuchContainer, exceptions.NotFound): raise HTTPNotFound(request=req) except exceptions.ClientException as err: # 481 = CODE_POLICY_NOT_SATISFIABLE if err.status == 481: raise exceptions.ServiceBusy() self.app.logger.exception( _('ERROR Exception transferring data %s'), {'path': req.path}) raise HTTPInternalServerError(request=req) except Exception: self.app.logger.exception( _('ERROR Exception transferring data %s'), {'path': req.path}) raise HTTPInternalServerError(request=req) resp = HTTPCreated(request=req, etag=checksum) return resp