def test_constrain_req_limit(self): req = Request.blank('') self.assertEqual(10, rh.constrain_req_limit(req, 10)) req = Request.blank('', query_string='limit=1') self.assertEqual(1, rh.constrain_req_limit(req, 10)) req = Request.blank('', query_string='limit=1.0') self.assertEqual(10, rh.constrain_req_limit(req, 10)) req = Request.blank('', query_string='limit=11') with self.assertRaises(HTTPException) as raised: rh.constrain_req_limit(req, 10) self.assertEqual(raised.exception.status_int, 412)
def GET(self, req): """Handle HTTP GET request.""" drive, part, account = get_account_name_and_placement(req) prefix = get_param(req, 'prefix') delimiter = get_param(req, 'delimiter') reverse = config_true_value(get_param(req, 'reverse')) limit = constrain_req_limit(req, constraints.ACCOUNT_LISTING_LIMIT) marker = get_param(req, 'marker', '') end_marker = get_param(req, 'end_marker') out_content_type = listing_formats.get_listing_content_type(req) try: check_drive(self.root, drive, self.mount_check) except ValueError: return HTTPInsufficientStorage(drive=drive, request=req) broker = self._get_account_broker(drive, part, account, pending_timeout=0.1, stale_reads_ok=True) if broker.is_deleted(): return self._deleted_response(broker, req, HTTPNotFound) return account_listing_response(account, req, out_content_type, broker, limit, marker, end_marker, prefix, delimiter, reverse)
def _get_from_shards(self, req, resp): # Construct listing using shards described by the response body. # The history of containers that have returned shard ranges is # maintained in the request environ so that loops can be avoided by # forcing an object listing if the same container is visited again. # This can happen in at least two scenarios: # 1. a container has filled a gap in its shard ranges with a # shard range pointing to itself # 2. a root container returns a (stale) shard range pointing to a # shard that has shrunk into the root, in which case the shrunken # shard may return the root's shard range. shard_listing_history = req.environ.setdefault( 'swift.shard_listing_history', []) shard_listing_history.append((self.account_name, self.container_name)) shard_ranges = [ ShardRange.from_dict(data) for data in json.loads(resp.body) ] self.app.logger.debug('GET listing from %s shards for: %s', len(shard_ranges), req.path_qs) if not shard_ranges: # can't find ranges or there was a problem getting the ranges. So # return what we have. return resp objects = [] req_limit = constrain_req_limit(req, CONTAINER_LISTING_LIMIT) params = req.params.copy() params.pop('states', None) req.headers.pop('X-Backend-Record-Type', None) reverse = config_true_value(params.get('reverse')) marker = wsgi_to_str(params.get('marker')) end_marker = wsgi_to_str(params.get('end_marker')) prefix = wsgi_to_str(params.get('prefix')) limit = req_limit for i, shard_range in enumerate(shard_ranges): params['limit'] = limit # Always set marker to ensure that object names less than or equal # to those already in the listing are not fetched; if the listing # is empty then the original request marker, if any, is used. This # allows misplaced objects below the expected shard range to be # included in the listing. if objects: last_name = objects[-1].get('name', objects[-1].get('subdir', u'')) params['marker'] = bytes_to_wsgi(last_name.encode('utf-8')) elif marker: params['marker'] = str_to_wsgi(marker) else: params['marker'] = '' # Always set end_marker to ensure that misplaced objects beyond the # expected shard range are not fetched. This prevents a misplaced # object obscuring correctly placed objects in the next shard # range. if end_marker and end_marker in shard_range: params['end_marker'] = str_to_wsgi(end_marker) elif reverse: params['end_marker'] = str_to_wsgi(shard_range.lower_str) else: params['end_marker'] = str_to_wsgi(shard_range.end_marker) headers = {} if ((shard_range.account, shard_range.container) in shard_listing_history): # directed back to same container - force GET of objects headers['X-Backend-Record-Type'] = 'object' if config_true_value(req.headers.get('x-newest', False)): headers['X-Newest'] = 'true' if prefix: if prefix > shard_range: continue try: just_past = prefix[:-1] + chr(ord(prefix[-1]) + 1) except ValueError: pass else: if just_past < shard_range: continue self.app.logger.debug( 'Getting listing part %d from shard %s %s with %s', i, shard_range, shard_range.name, headers) objs, shard_resp = self._get_container_listing( req, shard_range.account, shard_range.container, headers=headers, params=params) sharding_state = shard_resp.headers.get('x-backend-sharding-state', 'unknown') if objs is None: # tolerate errors self.app.logger.debug( 'Failed to get objects from shard (state=%s), total = %d', sharding_state, len(objects)) continue self.app.logger.debug( 'Found %d objects in shard (state=%s), total = %d', len(objs), sharding_state, len(objs) + len(objects)) if not objs: # tolerate empty shard containers continue objects.extend(objs) limit -= len(objs) if limit <= 0: break last_name = objects[-1].get('name', objects[-1].get('subdir', u'')) if six.PY2: last_name = last_name.encode('utf8') if end_marker and reverse and end_marker >= last_name: break if end_marker and not reverse and end_marker <= last_name: break resp.body = json.dumps(objects).encode('ascii') constrained = any( req.params.get(constraint) for constraint in ('marker', 'end_marker', 'path', 'prefix', 'delimiter')) if not constrained and len(objects) < req_limit: self.app.logger.debug('Setting object count to %s' % len(objects)) # prefer the actual listing stats over the potentially outdated # root stats. This condition is only likely when a sharded # container is shrinking or in tests; typically a sharded container # will have more than CONTAINER_LISTING_LIMIT objects so any # unconstrained listing will be capped by the limit and total # object stats cannot therefore be inferred from the listing. resp.headers['X-Container-Object-Count'] = len(objects) resp.headers['X-Container-Bytes-Used'] = sum( [o['bytes'] for o in objects]) return resp
def GET(self, req): """ Handle HTTP GET request. The body of the response to a successful GET request contains a listing of either objects or shard ranges. The exact content of the listing is determined by a combination of request headers and query string parameters, as follows: * The type of the listing is determined by the ``X-Backend-Record-Type`` header. If this header has value ``shard`` then the response body will be a list of shard ranges; if this header has value ``auto``, and the container state is ``sharding`` or ``sharded``, then the listing will be a list of shard ranges; otherwise the response body will be a list of objects. * Both shard range and object listings may be constrained to a name range by the ``marker`` and ``end_marker`` query string parameters. Object listings will only contain objects whose names are greater than any ``marker`` value and less than any ``end_marker`` value. Shard range listings will only contain shard ranges whose namespace is greater than or includes any ``marker`` value and is less than or includes any ``end_marker`` value. * Shard range listings may also be constrained by an ``includes`` query string parameter. If this parameter is present the listing will only contain shard ranges whose namespace includes the value of the parameter; any ``marker`` or ``end_marker`` parameters are ignored * The length of an object listing may be constrained by the ``limit`` parameter. Object listings may also be constrained by ``prefix``, ``delimiter`` and ``path`` query string parameters. * Shard range listings will include deleted shard ranges if and only if the ``X-Backend-Include-Deleted`` header value is one of :attr:`swift.common.utils.TRUE_VALUES`. Object listings never include deleted objects. * Shard range listings may be constrained to include only shard ranges whose state is specified by a query string ``states`` parameter. If present, the ``states`` parameter should be a comma separated list of either the string or integer representation of :data:`~swift.common.utils.ShardRange.STATES`. Two alias values may be used in a ``states`` parameter value: ``listing`` will cause the listing to include all shard ranges in a state suitable for contributing to an object listing; ``updating`` will cause the listing to include all shard ranges in a state suitable to accept an object update. If either of these aliases is used then the shard range listing will if necessary be extended with a synthesised 'filler' range in order to satisfy the requested name range when insufficient actual shard ranges are found. Any 'filler' shard range will cover the otherwise uncovered tail of the requested name range and will point back to the same container. * Listings are not normally returned from a deleted container. However, the ``X-Backend-Override-Deleted`` header may be used with a value in :attr:`swift.common.utils.TRUE_VALUES` to force a shard range listing to be returned from a deleted container whose DB file still exists. :param req: an instance of :class:`swift.common.swob.Request` :returns: an instance of :class:`swift.common.swob.Response` """ drive, part, account, container, obj = get_obj_name_and_placement(req) path = get_param(req, 'path') prefix = get_param(req, 'prefix') delimiter = get_param(req, 'delimiter') marker = get_param(req, 'marker', '') end_marker = get_param(req, 'end_marker') limit = constrain_req_limit(req, constraints.CONTAINER_LISTING_LIMIT) reverse = config_true_value(get_param(req, 'reverse')) out_content_type = listing_formats.get_listing_content_type(req) try: check_drive(self.root, drive, self.mount_check) except ValueError: return HTTPInsufficientStorage(drive=drive, request=req) broker = self._get_container_broker(drive, part, account, container, pending_timeout=0.1, stale_reads_ok=True) info, is_deleted = broker.get_info_is_deleted() record_type = req.headers.get('x-backend-record-type', '').lower() if record_type == 'auto' and info.get('db_state') in (SHARDING, SHARDED): record_type = 'shard' if record_type == 'shard': override_deleted = info and config_true_value( req.headers.get('x-backend-override-deleted', False)) resp_headers = gen_resp_headers(info, is_deleted=is_deleted and not override_deleted) if is_deleted and not override_deleted: return HTTPNotFound(request=req, headers=resp_headers) resp_headers['X-Backend-Record-Type'] = 'shard' includes = get_param(req, 'includes') states = get_param(req, 'states') fill_gaps = False if states: states = list_from_csv(states) fill_gaps = any(('listing' in states, 'updating' in states)) try: states = broker.resolve_shard_range_states(states) except ValueError: return HTTPBadRequest(request=req, body='Bad state') include_deleted = config_true_value( req.headers.get('x-backend-include-deleted', False)) container_list = broker.get_shard_ranges( marker, end_marker, includes, reverse, states=states, include_deleted=include_deleted, fill_gaps=fill_gaps) else: resp_headers = gen_resp_headers(info, is_deleted=is_deleted) if is_deleted: return HTTPNotFound(request=req, headers=resp_headers) resp_headers['X-Backend-Record-Type'] = 'object' # Use the retired db while container is in process of sharding, # otherwise use current db src_broker = broker.get_brokers()[0] container_list = src_broker.list_objects_iter( limit, marker, end_marker, prefix, delimiter, path, storage_policy_index=info['storage_policy_index'], reverse=reverse, allow_reserved=req.allow_reserved_names) return self.create_listing(req, out_content_type, info, resp_headers, broker.metadata, container_list, container)
def _list_versions(self, req, start_response, location): # Only supports JSON listings req.environ['swift.format_listing'] = False if not req.accept.best_match(['application/json']): raise HTTPNotAcceptable(request=req) params = req.params # FIXME(FVE) this is probably not working with oio-sds backend if 'version_marker' in params: if 'marker' not in params: raise HTTPBadRequest('version_marker param requires marker') if params['version_marker'] != 'null': try: ts = Timestamp(params.pop('version_marker')) except ValueError: raise HTTPBadRequest('invalid version_marker param') params['marker'] = self._build_versions_object_name( params['marker'], ts) delim = params.get('delimiter', '') # Exclude the set of chars used in version_id from user delimiters if set(delim).intersection('0123456789.%s' % RESERVED_STR): raise HTTPBadRequest('invalid delimiter param') null_listing = [] subdir_set = set() account = req.split_path(3, 3, True)[1] versions_req = make_pre_authed_request( req.environ, method='GET', swift_source='OV', path=wsgi_quote('/v1/%s/%s' % (account, location)), headers={'X-Backend-Allow-Reserved-Names': 'true'}, ) versions_req.environ['oio.query'] = {'versions': True} # NB: no end_marker support (yet) versions_req.params = { k: params.get(k, '') for k in ('prefix', 'marker', 'limit', 'delimiter', 'reverse', 'format') } versions_resp = versions_req.get_response(self.app) if versions_resp.status_int == HTTP_NOT_FOUND: raise versions_resp elif is_success(versions_resp.status_int): try: listing = json.loads(versions_resp.body) except ValueError: app_resp = [versions_resp.body] else: versions_listing = [] for item in listing: if 'subdir' in item: subdir_set.add(item['subdir']) else: item['version_id'] = str(item.get('version', 'null')) versions_listing.append(item) if (item['content_type'] == DELETE_MARKER_CONTENT_TYPE ): item['hash'] = MD5_OF_EMPTY_STRING subdir_listing = [{'subdir': s} for s in subdir_set] broken_listing = [] limit = constrain_req_limit(req, CONTAINER_LISTING_LIMIT) body = build_listing( null_listing, versions_listing, subdir_listing, broken_listing, reverse=config_true_value(params.get('reverse', 'no')), limit=limit, ) self.update_content_length(len(body)) app_resp = [body] else: return versions_resp(versions_req.environ, start_response) start_response(self._response_status, self._response_headers, self._response_exc_info) return app_resp