def GETorHEAD(self, req): had_match = False for match_header in ('if-match', 'if-none-match'): if match_header not in req.headers: continue had_match = True for value in list_from_csv(req.headers[match_header]): if value.startswith('"') and value.endswith('"'): value = value[1:-1] if value.endswith('-N'): # Deal with fake S3-like etags for SLOs uploaded via Swift req.headers[match_header] += ', ' + value[:-2] if had_match: # Update where to look update_etag_is_at_header(req, sysmeta_header('object', 'etag')) object_name = req.object_name version_id = req.params.get('versionId') if version_id and version_id != 'null': # get a specific version in the versioning container req.container_name += VERSIONING_SUFFIX req.object_name = versioned_object_name( req.object_name, req.params.pop('versionId')) cors_rule = None if req.headers.get('Origin'): cors_rule = get_cors(self.app, req, req.method, req.headers.get('Origin')) try: resp = req.get_response(self.app) except NoSuchKey: resp = None if version_id and version_id != 'null': # if the specific version is not in the versioning container, # it might be the current version req.container_name = req.container_name[ :-len(VERSIONING_SUFFIX)] req.object_name = object_name info = req.get_object_info(self.app, object_name=object_name) if info.get('sysmeta', {}).get('version-id') == version_id: resp = req.get_response(self.app) if resp is None: raise if req.method == 'HEAD': resp.app_iter = None if 'x-amz-meta-deleted' in resp.headers: raise NoSuchKey(object_name) for key in ('content-type', 'content-language', 'expires', 'cache-control', 'content-disposition', 'content-encoding'): if 'response-' + key in req.params: resp.headers[key] = req.params['response-' + key] if cors_rule is not None: cors_fill_headers(req, resp, cors_rule) return resp
def GETorHEAD(self, req): had_match = False for match_header in ('if-match', 'if-none-match'): if match_header not in req.headers: continue had_match = True for value in list_from_csv(req.headers[match_header]): value = normalize_etag(value) if value.endswith('-N'): # Deal with fake S3-like etags for SLOs uploaded via Swift req.headers[match_header] += ', ' + value[:-2] if had_match: # Update where to look update_etag_is_at_header(req, sysmeta_header('object', 'etag')) resp = req.get_response(self.app) if req.method == 'HEAD': resp.app_iter = None for key in ('content-type', 'content-language', 'expires', 'cache-control', 'content-disposition', 'content-encoding'): if 'response-' + key in req.params: resp.headers[key] = req.params['response-' + key] return resp
def handle_get_or_head(self, req, start_response): with self._mask_conditional_etags(req, 'If-Match') as masked1: with self._mask_conditional_etags(req, 'If-None-Match') as masked2: if masked1 or masked2: update_etag_is_at_header( req, 'X-Object-Sysmeta-Crypto-Etag-Mac') resp = self._app_call(req.environ) start_response(self._response_status, self._response_headers, self._response_exc_info) return resp
def GETorHEAD(self, req): had_match = False for match_header in ('if-match', 'if-none-match'): if match_header not in req.headers: continue had_match = True for value in list_from_csv(req.headers[match_header]): value = normalize_etag(value) if value.endswith('-N'): # Deal with fake S3-like etags for SLOs uploaded via Swift req.headers[match_header] += ', ' + value[:-2] if had_match: # Update where to look update_etag_is_at_header(req, sysmeta_header('object', 'etag')) object_name = req.object_name version_id = version_id_param(req) query = {} if version_id is None else {'version-id': version_id} if version_id not in ('null', None): container_info = req.get_container_info(self.app) if not container_info.get( 'sysmeta', {}).get('versions-container', ''): # Versioning has never been enabled raise NoSuchVersion(object_name, version_id) cors_rule = None if req.headers.get('Origin'): cors_rule = get_cors(self.app, req, req.method, req.headers.get('Origin')) resp = req.get_response(self.app, query=query) if req.method == 'HEAD': resp.app_iter = None if 'x-amz-meta-deleted' in resp.headers: raise NoSuchKey(object_name) for key in ('content-type', 'content-language', 'expires', 'cache-control', 'content-disposition', 'content-encoding'): if 'response-' + key in req.params: resp.headers[key] = req.params['response-' + key] if cors_rule is not None: cors_fill_headers(req, resp, cors_rule) return resp
def GETorHEAD(self, req): if any(match_header in req.headers for match_header in ('if-match', 'if-none-match')): # Update where to look update_etag_is_at_header(req, sysmeta_header('object', 'etag')) resp = req.get_response(self.app) if req.method == 'HEAD': resp.app_iter = None for key in ('content-type', 'content-language', 'expires', 'cache-control', 'content-disposition', 'content-encoding'): if 'response-' + key in req.params: resp.headers[key] = req.params['response-' + key] return resp
def GETorHEAD(self, req): had_match = False for match_header in ('if-match', 'if-none-match'): if match_header not in req.headers: continue had_match = True for value in list_from_csv(req.headers[match_header]): value = normalize_etag(value) if value.endswith('-N'): # Deal with fake S3-like etags for SLOs uploaded via Swift req.headers[match_header] += ', ' + value[:-2] if had_match: # Update where to look update_etag_is_at_header(req, sysmeta_header('object', 'etag')) object_name = req.object_name version_id = req.params.get('versionId') if version_id not in ('null', None) and \ 'object_versioning' not in get_swift_info(): raise S3NotImplemented() query = {} if version_id is None else {'version-id': version_id} resp = req.get_response(self.app, query=query) if req.method == 'HEAD': resp.app_iter = None if 'x-amz-meta-deleted' in resp.headers: raise NoSuchKey(object_name) for key in ('content-type', 'content-language', 'expires', 'cache-control', 'content-disposition', 'content-encoding'): if 'response-' + key in req.params: resp.headers[key] = req.params['response-' + key] return resp
def test_update_etag_is_at_header(self): # start with no existing X-Backend-Etag-Is-At req = Request.blank('/v/a/c/o') rh.update_etag_is_at_header(req, 'X-Object-Sysmeta-My-Etag') self.assertEqual('X-Object-Sysmeta-My-Etag', req.headers['X-Backend-Etag-Is-At']) # add another alternate rh.update_etag_is_at_header(req, 'X-Object-Sysmeta-Ec-Etag') self.assertEqual('X-Object-Sysmeta-My-Etag,X-Object-Sysmeta-Ec-Etag', req.headers['X-Backend-Etag-Is-At']) with self.assertRaises(ValueError) as cm: rh.update_etag_is_at_header(req, 'X-Object-Sysmeta-,-Bad') self.assertEqual('Header name must not contain commas', cm.exception.args[0])
def test_update_etag_is_at_header(self): # start with no existing X-Backend-Etag-Is-At req = Request.blank('/v/a/c/o') update_etag_is_at_header(req, 'X-Object-Sysmeta-My-Etag') self.assertEqual('X-Object-Sysmeta-My-Etag', req.headers['X-Backend-Etag-Is-At']) # add another alternate update_etag_is_at_header(req, 'X-Object-Sysmeta-Ec-Etag') self.assertEqual('X-Object-Sysmeta-My-Etag,X-Object-Sysmeta-Ec-Etag', req.headers['X-Backend-Etag-Is-At']) with self.assertRaises(ValueError) as cm: update_etag_is_at_header(req, 'X-Object-Sysmeta-,-Bad') self.assertEqual('Header name must not contain commas', cm.exception.message)
def handle_slo_get_or_head(self, req, start_response): """ Takes a request and a start_response callable and does the normal WSGI thing with them. Returns an iterator suitable for sending up the WSGI chain. :param req: swob.Request object; is a GET or HEAD request aimed at what may be a static large object manifest (or may not). :param start_response: WSGI start_response callable """ if req.params.get('multipart-manifest') != 'get': # If this object is an SLO manifest, we may have saved off the # large object etag during the original PUT. Send an # X-Backend-Etag-Is-At header so that, if the SLO etag *was* # saved, we can trust the object-server to respond appropriately # to If-Match/If-None-Match requests. update_etag_is_at_header(req, SYSMETA_SLO_ETAG) resp_iter = self._app_call(req.environ) # make sure this response is for a static large object manifest slo_marker = slo_etag = slo_size = None for header, value in self._response_headers: header = header.lower() if header == SYSMETA_SLO_ETAG: slo_etag = value elif header == SYSMETA_SLO_SIZE: slo_size = value elif (header == 'x-static-large-object' and config_true_value(value)): slo_marker = value if slo_marker and slo_etag and slo_size: break if not slo_marker: # Not a static large object manifest. Just pass it through. start_response(self._response_status, self._response_headers, self._response_exc_info) return resp_iter # Handle pass-through request for the manifest itself if req.params.get('multipart-manifest') == 'get': if req.params.get('format') == 'raw': resp_iter = self.convert_segment_listing( self._response_headers, resp_iter) else: new_headers = [] for header, value in self._response_headers: if header.lower() == 'content-type': new_headers.append(('Content-Type', 'application/json; charset=utf-8')) else: new_headers.append((header, value)) self._response_headers = new_headers start_response(self._response_status, self._response_headers, self._response_exc_info) return resp_iter is_conditional = self._response_status.startswith( ('304', '412')) and (req.if_match or req.if_none_match) if slo_etag and slo_size and (req.method == 'HEAD' or is_conditional): # Since we have length and etag, we can respond immediately for i, (header, _value) in enumerate(self._response_headers): lheader = header.lower() if lheader == 'etag': self._response_headers[i] = (header, '"%s"' % slo_etag) elif lheader == 'content-length' and not is_conditional: self._response_headers[i] = (header, slo_size) start_response(self._response_status, self._response_headers, self._response_exc_info) return resp_iter if self._need_to_refetch_manifest(req): req.environ['swift.non_client_disconnect'] = True close_if_possible(resp_iter) del req.environ['swift.non_client_disconnect'] get_req = make_subrequest( req.environ, method='GET', headers={'x-auth-token': req.headers.get('x-auth-token')}, agent='%(orig)s SLO MultipartGET', swift_source='SLO') resp_iter = self._app_call(get_req.environ) # Any Content-Range from a manifest is almost certainly wrong for the # full large object. resp_headers = [(h, v) for h, v in self._response_headers if not h.lower() == 'content-range'] response = self.get_or_head_response(req, resp_headers, resp_iter) return response(req.environ, start_response)
def handle_slo_get_or_head(self, req, start_response): """ Takes a request and a start_response callable and does the normal WSGI thing with them. Returns an iterator suitable for sending up the WSGI chain. :param req: swob.Request object; is a GET or HEAD request aimed at what may be a static large object manifest (or may not). :param start_response: WSGI start_response callable """ if req.params.get('multipart-manifest') != 'get': # If this object is an SLO manifest, we may have saved off the # large object etag during the original PUT. Send an # X-Backend-Etag-Is-At header so that, if the SLO etag *was* # saved, we can trust the object-server to respond appropriately # to If-Match/If-None-Match requests. update_etag_is_at_header(req, SYSMETA_SLO_ETAG) resp_iter = self._app_call(req.environ) # make sure this response is for a static large object manifest slo_marker = slo_etag = slo_size = None for header, value in self._response_headers: header = header.lower() if header == SYSMETA_SLO_ETAG: slo_etag = value elif header == SYSMETA_SLO_SIZE: slo_size = value elif (header == 'x-static-large-object' and config_true_value(value)): slo_marker = value if slo_marker and slo_etag and slo_size: break if not slo_marker: # Not a static large object manifest. Just pass it through. start_response(self._response_status, self._response_headers, self._response_exc_info) return resp_iter # Handle pass-through request for the manifest itself if req.params.get('multipart-manifest') == 'get': if req.params.get('format') == 'raw': resp_iter = self.convert_segment_listing( self._response_headers, resp_iter) else: new_headers = [] for header, value in self._response_headers: if header.lower() == 'content-type': new_headers.append(('Content-Type', 'application/json; charset=utf-8')) else: new_headers.append((header, value)) self._response_headers = new_headers start_response(self._response_status, self._response_headers, self._response_exc_info) return resp_iter is_conditional = self._response_status.startswith(('304', '412')) and ( req.if_match or req.if_none_match) if slo_etag and slo_size and ( req.method == 'HEAD' or is_conditional): # Since we have length and etag, we can respond immediately for i, (header, _value) in enumerate(self._response_headers): lheader = header.lower() if lheader == 'etag': self._response_headers[i] = (header, '"%s"' % slo_etag) elif lheader == 'content-length' and not is_conditional: self._response_headers[i] = (header, slo_size) start_response(self._response_status, self._response_headers, self._response_exc_info) return resp_iter if self._need_to_refetch_manifest(req): req.environ['swift.non_client_disconnect'] = True close_if_possible(resp_iter) del req.environ['swift.non_client_disconnect'] get_req = make_subrequest( req.environ, method='GET', headers={'x-auth-token': req.headers.get('x-auth-token')}, agent='%(orig)s SLO MultipartGET', swift_source='SLO') resp_iter = self._app_call(get_req.environ) # Any Content-Range from a manifest is almost certainly wrong for the # full large object. resp_headers = [(h, v) for h, v in self._response_headers if not h.lower() == 'content-range'] response = self.get_or_head_response( req, resp_headers, resp_iter) return response(req.environ, start_response)
def GETorHEAD(self, req): """ Handled GET or HEAD request on a part of a multipart object. """ part_number = self.parse_part_number(req) had_match = False for match_header in ('if-match', 'if-none-match'): if match_header not in req.headers: continue had_match = True for value in list_from_csv(req.headers[match_header]): if value.startswith('"') and value.endswith('"'): value = value[1:-1] if value.endswith('-N'): # Deal with fake S3-like etags for SLOs uploaded via Swift req.headers[match_header] += ', ' + value[:-2] if had_match: # Update where to look update_etag_is_at_header(req, sysmeta_header('object', 'etag')) # Get the list of parts. Must be raw to get all response headers. slo_resp = req.get_response(self.app, 'GET', req.container_name, req.object_name, query={ 'multipart-manifest': 'get', 'format': 'raw' }) # Check if the object is really a SLO. If not, and user asked # for the first part, do a regular request. if 'X-Static-Large-Object' not in slo_resp.sw_headers: if part_number == 1: if slo_resp.is_success and req.method == 'HEAD': # Clear body slo_resp.body = '' return slo_resp else: close_if_possible(slo_resp.app_iter) raise InvalidRange() # Locate the part slo = json.loads(slo_resp.body) try: part = slo[part_number - 1] except IndexError: raise InvalidRange() # Redirect the request on the part _, req.container_name, req.object_name = part['path'].split('/', 2) # XXX enforce container_name and object_name to be <str> # or it will rise issues in swift3/requests when merging both req.container_name = req.container_name.encode('utf-8') req.object_name = req.object_name.encode('utf8') # The etag check was performed with the manifest if had_match: for match_header in ('if-match', 'if-none-match'): req.headers.pop(match_header, None) resp = req.get_response(self.app) # Replace status slo_resp.status = resp.status # Replace body slo_resp.app_iter = resp.app_iter # Update with the size of the part slo_resp.headers['Content-Length'] = \ resp.headers.get('Content-Length', 0) slo_resp.sw_headers['Content-Length'] = \ slo_resp.headers['Content-Length'] # Add the number of parts in this object slo_resp.headers['X-Amz-Mp-Parts-Count'] = len(slo) return slo_resp