def _manifest_head_response(self, req, response_headers): conditional_etag = resolve_etag_is_at_header(req, response_headers) return HTTPOk(request=req, headers=response_headers, body='', conditional_etag=conditional_etag, conditional_response=True)
def do_test(): req = Request.blank('/v/a/c/o') # ok to have no X-Backend-Etag-Is-At self.assertIsNone(rh.resolve_etag_is_at_header(req, metadata)) # ok to have no matching metadata req.headers['X-Backend-Etag-Is-At'] = 'X-Not-There' self.assertIsNone(rh.resolve_etag_is_at_header(req, metadata)) # selects from metadata req.headers['X-Backend-Etag-Is-At'] = 'X-Object-Sysmeta-Ec-Etag' self.assertEqual('an etag value', rh.resolve_etag_is_at_header(req, metadata)) req.headers['X-Backend-Etag-Is-At'] = 'X-Object-Sysmeta-My-Etag' self.assertEqual('another etag value', rh.resolve_etag_is_at_header(req, metadata)) # first in list takes precedence req.headers['X-Backend-Etag-Is-At'] = \ 'X-Object-Sysmeta-My-Etag,X-Object-Sysmeta-Ec-Etag' self.assertEqual('another etag value', rh.resolve_etag_is_at_header(req, metadata)) # non-existent alternates are passed over req.headers['X-Backend-Etag-Is-At'] = \ 'X-Bogus,X-Object-Sysmeta-My-Etag,X-Object-Sysmeta-Ec-Etag' self.assertEqual('another etag value', rh.resolve_etag_is_at_header(req, metadata)) # spaces in list are ok alts = 'X-Foo, X-Object-Sysmeta-My-Etag , X-Object-Sysmeta-Ec-Etag' req.headers['X-Backend-Etag-Is-At'] = alts self.assertEqual('another etag value', rh.resolve_etag_is_at_header(req, metadata)) # lower case in list is ok alts = alts.lower() req.headers['X-Backend-Etag-Is-At'] = alts self.assertEqual('another etag value', rh.resolve_etag_is_at_header(req, metadata)) # upper case in list is ok alts = alts.upper() req.headers['X-Backend-Etag-Is-At'] = alts self.assertEqual('another etag value', rh.resolve_etag_is_at_header(req, metadata))
def do_test(): req = Request.blank('/v/a/c/o') # ok to have no X-Backend-Etag-Is-At self.assertIsNone(resolve_etag_is_at_header(req, metadata)) # ok to have no matching metadata req.headers['X-Backend-Etag-Is-At'] = 'X-Not-There' self.assertIsNone(resolve_etag_is_at_header(req, metadata)) # selects from metadata req.headers['X-Backend-Etag-Is-At'] = 'X-Object-Sysmeta-Ec-Etag' self.assertEqual('an etag value', resolve_etag_is_at_header(req, metadata)) req.headers['X-Backend-Etag-Is-At'] = 'X-Object-Sysmeta-My-Etag' self.assertEqual('another etag value', resolve_etag_is_at_header(req, metadata)) # first in list takes precedence req.headers['X-Backend-Etag-Is-At'] = \ 'X-Object-Sysmeta-My-Etag,X-Object-Sysmeta-Ec-Etag' self.assertEqual('another etag value', resolve_etag_is_at_header(req, metadata)) # non-existent alternates are passed over req.headers['X-Backend-Etag-Is-At'] = \ 'X-Bogus,X-Object-Sysmeta-My-Etag,X-Object-Sysmeta-Ec-Etag' self.assertEqual('another etag value', resolve_etag_is_at_header(req, metadata)) # spaces in list are ok alts = 'X-Foo, X-Object-Sysmeta-My-Etag , X-Object-Sysmeta-Ec-Etag' req.headers['X-Backend-Etag-Is-At'] = alts self.assertEqual('another etag value', resolve_etag_is_at_header(req, metadata)) # lower case in list is ok alts = alts.lower() req.headers['X-Backend-Etag-Is-At'] = alts self.assertEqual('another etag value', resolve_etag_is_at_header(req, metadata)) # upper case in list is ok alts = alts.upper() req.headers['X-Backend-Etag-Is-At'] = alts self.assertEqual('another etag value', resolve_etag_is_at_header(req, metadata))
def make_object_response(self, req, metadata, stream=None): conditional_etag = resolve_etag_is_at_header( req, metadata.get('properties')) resp = Response(request=req, conditional_response=True, conditional_etag=conditional_etag) if config_true_value(metadata['deleted']): resp.headers['Content-Type'] = DELETE_MARKER_CONTENT_TYPE else: resp.headers['Content-Type'] = metadata.get( 'mime_type', 'application/octet-stream') properties = metadata.get('properties') if properties: for k, v in properties.iteritems(): if is_sys_or_user_meta('object', k) or \ is_object_transient_sysmeta(k) or \ k.lower() in self.allowed_headers: resp.headers[str(k)] = v hash_ = metadata.get('hash') if hash_ is not None: hash_ = hash_.lower() resp.headers['etag'] = hash_ resp.headers['x-object-sysmeta-version-id'] = metadata['version'] resp.last_modified = int(metadata['mtime']) if stream: # Whether we are bothered with ranges or not, we wrap the # stream in order to handle exceptions. resp.app_iter = StreamRangeIterator(req, stream) length_ = metadata.get('length') if length_ is not None: length_ = int(length_) resp.content_length = length_ resp.content_encoding = metadata.get('encoding') resp.accept_ranges = 'bytes' return resp
def __call__(self, env, start_response): method = env['REQUEST_METHOD'] if method not in self.ALLOWED_METHODS: raise HTTPNotImplemented() path = env['PATH_INFO'] _, acc, cont, obj = split_path(env['PATH_INFO'], 0, 4, rest_with_last=True) if env.get('QUERY_STRING'): path += '?' + env['QUERY_STRING'] if 'swift.authorize' in env: resp = env['swift.authorize'](swob.Request(env)) if resp: return resp(env, start_response) req = swob.Request(env) self.swift_sources.append(env.get('swift.source')) self.txn_ids.append(env.get('swift.trans_id')) try: resp_class, raw_headers, body = self._find_response(method, path) headers = HeaderKeyDict(raw_headers) except KeyError: if (env.get('QUERY_STRING') and (method, env['PATH_INFO']) in self._responses): resp_class, raw_headers, body = self._find_response( method, env['PATH_INFO']) headers = HeaderKeyDict(raw_headers) elif method == 'HEAD' and ('GET', path) in self._responses: resp_class, raw_headers, body = self._find_response( 'GET', path) body = None headers = HeaderKeyDict(raw_headers) elif method == 'GET' and obj and path in self.uploaded: resp_class = swob.HTTPOk headers, body = self.uploaded[path] else: raise KeyError("Didn't find %r in allowed responses" % ((method, path), )) # simulate object PUT if method == 'PUT' and obj: put_body = ''.join(iter(env['wsgi.input'].read, '')) if 'swift.callback.update_footers' in env: footers = HeaderKeyDict() env['swift.callback.update_footers'](footers) req.headers.update(footers) etag = md5(put_body).hexdigest() headers.setdefault('Etag', etag) headers.setdefault('Content-Length', len(put_body)) # keep it for subsequent GET requests later self.uploaded[path] = (dict(req.headers), put_body) if "CONTENT_TYPE" in env: self.uploaded[path][0]['Content-Type'] = env["CONTENT_TYPE"] # simulate object POST elif method == 'POST' and obj: metadata, data = self.uploaded.get(path, ({}, None)) # select items to keep from existing... new_metadata = dict((k, v) for k, v in metadata.items() if (not is_user_meta('object', k) and not is_object_transient_sysmeta(k))) # apply from new new_metadata.update( dict((k, v) for k, v in req.headers.items() if ( is_user_meta('object', k) or is_object_transient_sysmeta(k) or k.lower == 'content-type'))) self.uploaded[path] = new_metadata, data # note: tests may assume this copy of req_headers is case insensitive # so we deliberately use a HeaderKeyDict self._calls.append( FakeSwiftCall(method, path, HeaderKeyDict(req.headers))) # Apply conditional etag overrides conditional_etag = resolve_etag_is_at_header(req, headers) # range requests ought to work, hence conditional_response=True if isinstance(body, list): resp = resp_class(req=req, headers=headers, app_iter=body, conditional_response=req.method in ('GET', 'HEAD'), conditional_etag=conditional_etag) else: resp = resp_class(req=req, headers=headers, body=body, conditional_response=req.method in ('GET', 'HEAD'), conditional_etag=conditional_etag) wsgi_iter = resp(env, start_response) self.mark_opened(path) return LeakTrackingIter(wsgi_iter, self.mark_closed, path)
def __call__(self, env, start_response): method = env['REQUEST_METHOD'] if method not in self.ALLOWED_METHODS: raise HTTPNotImplemented() path = env['PATH_INFO'] _, acc, cont, obj = split_path(env['PATH_INFO'], 0, 4, rest_with_last=True) if env.get('QUERY_STRING'): path += '?' + env['QUERY_STRING'] if 'swift.authorize' in env: resp = env['swift.authorize'](swob.Request(env)) if resp: return resp(env, start_response) req = swob.Request(env) self.swift_sources.append(env.get('swift.source')) self.txn_ids.append(env.get('swift.trans_id')) try: resp_class, raw_headers, body = self._find_response(method, path) headers = HeaderKeyDict(raw_headers) except KeyError: if (env.get('QUERY_STRING') and (method, env['PATH_INFO']) in self._responses): resp_class, raw_headers, body = self._find_response( method, env['PATH_INFO']) headers = HeaderKeyDict(raw_headers) elif method == 'HEAD' and ('GET', path) in self._responses: resp_class, raw_headers, body = self._find_response( 'GET', path) body = None headers = HeaderKeyDict(raw_headers) elif method == 'GET' and obj and path in self.uploaded: resp_class = swob.HTTPOk headers, body = self.uploaded[path] else: raise KeyError("Didn't find %r in allowed responses" % ( (method, path),)) # simulate object PUT if method == 'PUT' and obj: put_body = b''.join(iter(env['wsgi.input'].read, b'')) if 'swift.callback.update_footers' in env: footers = HeaderKeyDict() env['swift.callback.update_footers'](footers) req.headers.update(footers) etag = md5(put_body).hexdigest() headers.setdefault('Etag', etag) headers.setdefault('Content-Length', len(put_body)) # keep it for subsequent GET requests later self.uploaded[path] = (dict(req.headers), put_body) if "CONTENT_TYPE" in env: self.uploaded[path][0]['Content-Type'] = env["CONTENT_TYPE"] # simulate object POST elif method == 'POST' and obj: metadata, data = self.uploaded.get(path, ({}, None)) # select items to keep from existing... new_metadata = dict( (k, v) for k, v in metadata.items() if (not is_user_meta('object', k) and not is_object_transient_sysmeta(k))) # apply from new new_metadata.update( dict((k, v) for k, v in req.headers.items() if (is_user_meta('object', k) or is_object_transient_sysmeta(k) or k.lower == 'content-type'))) self.uploaded[path] = new_metadata, data # note: tests may assume this copy of req_headers is case insensitive # so we deliberately use a HeaderKeyDict self._calls.append( FakeSwiftCall(method, path, HeaderKeyDict(req.headers))) # Apply conditional etag overrides conditional_etag = resolve_etag_is_at_header(req, headers) # range requests ought to work, hence conditional_response=True if isinstance(body, list): resp = resp_class( req=req, headers=headers, app_iter=body, conditional_response=req.method in ('GET', 'HEAD'), conditional_etag=conditional_etag) else: resp = resp_class( req=req, headers=headers, body=body, conditional_response=req.method in ('GET', 'HEAD'), conditional_etag=conditional_etag) wsgi_iter = resp(env, start_response) self.mark_opened(path) return LeakTrackingIter(wsgi_iter, self.mark_closed, path)
def handle_slo_get_or_head(self, req, start_response): """ Takes a request and a start_response callable and does the normal WSGI thing with them. Returns an iterator suitable for sending up the WSGI chain. :param req: :class:`~swift.common.swob.Request` object; is a ``GET`` or ``HEAD`` request aimed at what may (or may not) be a static large object manifest. :param start_response: WSGI start_response callable """ if req.params.get('multipart-manifest') != 'get': # If this object is an SLO manifest, we may have saved off the # large object etag during the original PUT. Send an # X-Backend-Etag-Is-At header so that, if the SLO etag *was* # saved, we can trust the object-server to respond appropriately # to If-Match/If-None-Match requests. update_etag_is_at_header(req, SYSMETA_SLO_ETAG) resp_iter = self._app_call(req.environ) # make sure this response is for a static large object manifest slo_marker = slo_etag = slo_size = None for header, value in self._response_headers: header = header.lower() if header == SYSMETA_SLO_ETAG: slo_etag = value elif header == SYSMETA_SLO_SIZE: slo_size = value elif (header == 'x-static-large-object' and config_true_value(value)): slo_marker = value if slo_marker and slo_etag and slo_size: break if not slo_marker: # Not a static large object manifest. Just pass it through. start_response(self._response_status, self._response_headers, self._response_exc_info) return resp_iter # Handle pass-through request for the manifest itself if req.params.get('multipart-manifest') == 'get': if req.params.get('format') == 'raw': resp_iter = self.convert_segment_listing( self._response_headers, resp_iter) else: new_headers = [] for header, value in self._response_headers: if header.lower() == 'content-type': new_headers.append(('Content-Type', 'application/json; charset=utf-8')) else: new_headers.append((header, value)) self._response_headers = new_headers start_response(self._response_status, self._response_headers, self._response_exc_info) return resp_iter is_conditional = self._response_status.startswith( ('304', '412')) and (req.if_match or req.if_none_match) if slo_etag and slo_size and (req.method == 'HEAD' or is_conditional): # Since we have length and etag, we can respond immediately resp = Response(status=self._response_status, headers=self._response_headers, app_iter=resp_iter, request=req, conditional_etag=resolve_etag_is_at_header( req, self._response_headers), conditional_response=True) resp.headers.update({ 'Etag': '"%s"' % slo_etag, 'Content-Length': slo_size, }) return resp(req.environ, start_response) if self._need_to_refetch_manifest(req): req.environ['swift.non_client_disconnect'] = True close_if_possible(resp_iter) del req.environ['swift.non_client_disconnect'] get_req = make_subrequest( req.environ, method='GET', headers={'x-auth-token': req.headers.get('x-auth-token')}, agent='%(orig)s SLO MultipartGET', swift_source='SLO') resp_iter = self._app_call(get_req.environ) # Any Content-Range from a manifest is almost certainly wrong for the # full large object. resp_headers = [(h, v) for h, v in self._response_headers if not h.lower() == 'content-range'] response = self.get_or_head_response(req, resp_headers, resp_iter) return response(req.environ, start_response)
def _manifest_get_response(self, req, content_length, response_headers, segments): if req.range: byteranges = [ # For some reason, swob.Range.ranges_for_length adds 1 to the # last byte's position. (start, end - 1) for start, end in req.range.ranges_for_length(content_length) ] else: byteranges = [] ver, account, _junk = req.split_path(3, 3, rest_with_last=True) plain_listing_iter = self._segment_listing_iterator( req, ver, account, segments, byteranges) def ratelimit_predicate(seg_dict): if 'raw_data' in seg_dict: return False # it's already in memory anyway start = seg_dict.get('start_byte') or 0 end = seg_dict.get('end_byte') if end is None: end = int(seg_dict['bytes']) - 1 is_small = (end - start + 1) < self.slo.rate_limit_under_size return is_small ratelimited_listing_iter = RateLimitedIterator( plain_listing_iter, self.slo.rate_limit_segments_per_sec, limit_after=self.slo.rate_limit_after_segment, ratelimit_if=ratelimit_predicate) # data segments are already in the correct format, but object-backed # segments need a path key added segment_listing_iter = (seg_dict if 'raw_data' in seg_dict else dict( seg_dict, path=self._segment_path(ver, account, seg_dict)) for seg_dict in ratelimited_listing_iter) segmented_iter = SegmentedIterable(req, self.slo.app, segment_listing_iter, name=req.path, logger=self.slo.logger, ua_suffix="SLO MultipartGET", swift_source="SLO", max_get_time=self.slo.max_get_time) try: segmented_iter.validate_first_segment() except (ListingIterError, SegmentError): # Copy from the SLO explanation in top of this file. # If any of the segments from the manifest are not found or # their Etag/Content Length no longer match the connection # will drop. In this case a 409 Conflict will be logged in # the proxy logs and the user will receive incomplete results. return HTTPConflict(request=req) conditional_etag = resolve_etag_is_at_header(req, response_headers) response = Response(request=req, content_length=content_length, headers=response_headers, conditional_response=True, conditional_etag=conditional_etag, app_iter=segmented_iter) return response