def GETorHEAD(self, req):
had_match = False
for match_header in ('if-match', 'if-none-match'):
if match_header not in req.headers:
continue
had_match = True
for value in list_from_csv(req.headers[match_header]):
if value.startswith('"') and value.endswith('"'):
value = value[1:-1]
if value.endswith('-N'):
# Deal with fake S3-like etags for SLOs uploaded via Swift
req.headers[match_header] += ', ' + value[:-2]
if had_match:
# Update where to look
update_etag_is_at_header(req, sysmeta_header('object', 'etag'))
object_name = req.object_name
version_id = req.params.get('versionId')
if version_id and version_id != 'null':
# get a specific version in the versioning container
req.container_name += VERSIONING_SUFFIX
req.object_name = versioned_object_name(
req.object_name, req.params.pop('versionId'))
cors_rule = None
if req.headers.get('Origin'):
cors_rule = get_cors(self.app, req, req.method,
req.headers.get('Origin'))
try:
resp = req.get_response(self.app)
except NoSuchKey:
resp = None
if version_id and version_id != 'null':
# if the specific version is not in the versioning container,
# it might be the current version
req.container_name = req.container_name[
:-len(VERSIONING_SUFFIX)]
req.object_name = object_name
info = req.get_object_info(self.app, object_name=object_name)
if info.get('sysmeta', {}).get('version-id') == version_id:
resp = req.get_response(self.app)
if resp is None:
raise
if req.method == 'HEAD':
resp.app_iter = None
if 'x-amz-meta-deleted' in resp.headers:
raise NoSuchKey(object_name)
for key in ('content-type', 'content-language', 'expires',
'cache-control', 'content-disposition',
'content-encoding'):
if 'response-' + key in req.params:
resp.headers[key] = req.params['response-' + key]
if cors_rule is not None:
cors_fill_headers(req, resp, cors_rule)
return resp
def GETorHEAD(self, req):
had_match = False
for match_header in ('if-match', 'if-none-match'):
if match_header not in req.headers:
continue
had_match = True
for value in list_from_csv(req.headers[match_header]):
value = normalize_etag(value)
if value.endswith('-N'):
# Deal with fake S3-like etags for SLOs uploaded via Swift
req.headers[match_header] += ', ' + value[:-2]
if had_match:
# Update where to look
update_etag_is_at_header(req, sysmeta_header('object', 'etag'))
resp = req.get_response(self.app)
if req.method == 'HEAD':
resp.app_iter = None
for key in ('content-type', 'content-language', 'expires',
'cache-control', 'content-disposition',
'content-encoding'):
if 'response-' + key in req.params:
resp.headers[key] = req.params['response-' + key]
return resp
def handle_get_or_head(self, req, start_response):
with self._mask_conditional_etags(req, 'If-Match') as masked1:
with self._mask_conditional_etags(req, 'If-None-Match') as masked2:
if masked1 or masked2:
update_etag_is_at_header(
req, 'X-Object-Sysmeta-Crypto-Etag-Mac')
resp = self._app_call(req.environ)
start_response(self._response_status, self._response_headers,
self._response_exc_info)
return resp
def handle_get_or_head(self, req, start_response):
with self._mask_conditional_etags(req, 'If-Match') as masked1:
with self._mask_conditional_etags(req, 'If-None-Match') as masked2:
if masked1 or masked2:
update_etag_is_at_header(
req, 'X-Object-Sysmeta-Crypto-Etag-Mac')
resp = self._app_call(req.environ)
start_response(self._response_status, self._response_headers,
self._response_exc_info)
return resp
def GETorHEAD(self, req):
had_match = False
for match_header in ('if-match', 'if-none-match'):
if match_header not in req.headers:
continue
had_match = True
for value in list_from_csv(req.headers[match_header]):
value = normalize_etag(value)
if value.endswith('-N'):
# Deal with fake S3-like etags for SLOs uploaded via Swift
req.headers[match_header] += ', ' + value[:-2]
if had_match:
# Update where to look
update_etag_is_at_header(req, sysmeta_header('object', 'etag'))
object_name = req.object_name
version_id = version_id_param(req)
query = {} if version_id is None else {'version-id': version_id}
if version_id not in ('null', None):
container_info = req.get_container_info(self.app)
if not container_info.get(
'sysmeta', {}).get('versions-container', ''):
# Versioning has never been enabled
raise NoSuchVersion(object_name, version_id)
cors_rule = None
if req.headers.get('Origin'):
cors_rule = get_cors(self.app, req, req.method,
req.headers.get('Origin'))
resp = req.get_response(self.app, query=query)
if req.method == 'HEAD':
resp.app_iter = None
if 'x-amz-meta-deleted' in resp.headers:
raise NoSuchKey(object_name)
for key in ('content-type', 'content-language', 'expires',
'cache-control', 'content-disposition',
'content-encoding'):
if 'response-' + key in req.params:
resp.headers[key] = req.params['response-' + key]
if cors_rule is not None:
cors_fill_headers(req, resp, cors_rule)
return resp
def GETorHEAD(self, req):
if any(match_header in req.headers
for match_header in ('if-match', 'if-none-match')):
# Update where to look
update_etag_is_at_header(req, sysmeta_header('object', 'etag'))
resp = req.get_response(self.app)
if req.method == 'HEAD':
resp.app_iter = None
for key in ('content-type', 'content-language', 'expires',
'cache-control', 'content-disposition',
'content-encoding'):
if 'response-' + key in req.params:
resp.headers[key] = req.params['response-' + key]
return resp
def GETorHEAD(self, req):
had_match = False
for match_header in ('if-match', 'if-none-match'):
if match_header not in req.headers:
continue
had_match = True
for value in list_from_csv(req.headers[match_header]):
value = normalize_etag(value)
if value.endswith('-N'):
# Deal with fake S3-like etags for SLOs uploaded via Swift
req.headers[match_header] += ', ' + value[:-2]
if had_match:
# Update where to look
update_etag_is_at_header(req, sysmeta_header('object', 'etag'))
object_name = req.object_name
version_id = req.params.get('versionId')
if version_id not in ('null', None) and \
'object_versioning' not in get_swift_info():
raise S3NotImplemented()
query = {} if version_id is None else {'version-id': version_id}
resp = req.get_response(self.app, query=query)
if req.method == 'HEAD':
resp.app_iter = None
if 'x-amz-meta-deleted' in resp.headers:
raise NoSuchKey(object_name)
for key in ('content-type', 'content-language', 'expires',
'cache-control', 'content-disposition',
'content-encoding'):
if 'response-' + key in req.params:
resp.headers[key] = req.params['response-' + key]
return resp
def test_update_etag_is_at_header(self):
# start with no existing X-Backend-Etag-Is-At
req = Request.blank('/v/a/c/o')
rh.update_etag_is_at_header(req, 'X-Object-Sysmeta-My-Etag')
self.assertEqual('X-Object-Sysmeta-My-Etag',
req.headers['X-Backend-Etag-Is-At'])
# add another alternate
rh.update_etag_is_at_header(req, 'X-Object-Sysmeta-Ec-Etag')
self.assertEqual('X-Object-Sysmeta-My-Etag,X-Object-Sysmeta-Ec-Etag',
req.headers['X-Backend-Etag-Is-At'])
with self.assertRaises(ValueError) as cm:
rh.update_etag_is_at_header(req, 'X-Object-Sysmeta-,-Bad')
self.assertEqual('Header name must not contain commas',
cm.exception.args[0])
def test_update_etag_is_at_header(self):
# start with no existing X-Backend-Etag-Is-At
req = Request.blank('/v/a/c/o')
update_etag_is_at_header(req, 'X-Object-Sysmeta-My-Etag')
self.assertEqual('X-Object-Sysmeta-My-Etag',
req.headers['X-Backend-Etag-Is-At'])
# add another alternate
update_etag_is_at_header(req, 'X-Object-Sysmeta-Ec-Etag')
self.assertEqual('X-Object-Sysmeta-My-Etag,X-Object-Sysmeta-Ec-Etag',
req.headers['X-Backend-Etag-Is-At'])
with self.assertRaises(ValueError) as cm:
update_etag_is_at_header(req, 'X-Object-Sysmeta-,-Bad')
self.assertEqual('Header name must not contain commas',
cm.exception.message)
def handle_slo_get_or_head(self, req, start_response):
"""
Takes a request and a start_response callable and does the normal WSGI
thing with them. Returns an iterator suitable for sending up the WSGI
chain.
:param req: swob.Request object; is a GET or HEAD request aimed at
what may be a static large object manifest (or may not).
:param start_response: WSGI start_response callable
"""
if req.params.get('multipart-manifest') != 'get':
# If this object is an SLO manifest, we may have saved off the
# large object etag during the original PUT. Send an
# X-Backend-Etag-Is-At header so that, if the SLO etag *was*
# saved, we can trust the object-server to respond appropriately
# to If-Match/If-None-Match requests.
update_etag_is_at_header(req, SYSMETA_SLO_ETAG)
resp_iter = self._app_call(req.environ)
# make sure this response is for a static large object manifest
slo_marker = slo_etag = slo_size = None
for header, value in self._response_headers:
header = header.lower()
if header == SYSMETA_SLO_ETAG:
slo_etag = value
elif header == SYSMETA_SLO_SIZE:
slo_size = value
elif (header == 'x-static-large-object'
and config_true_value(value)):
slo_marker = value
if slo_marker and slo_etag and slo_size:
break
if not slo_marker:
# Not a static large object manifest. Just pass it through.
start_response(self._response_status, self._response_headers,
self._response_exc_info)
return resp_iter
# Handle pass-through request for the manifest itself
if req.params.get('multipart-manifest') == 'get':
if req.params.get('format') == 'raw':
resp_iter = self.convert_segment_listing(
self._response_headers, resp_iter)
else:
new_headers = []
for header, value in self._response_headers:
if header.lower() == 'content-type':
new_headers.append(('Content-Type',
'application/json; charset=utf-8'))
else:
new_headers.append((header, value))
self._response_headers = new_headers
start_response(self._response_status, self._response_headers,
self._response_exc_info)
return resp_iter
is_conditional = self._response_status.startswith(
('304', '412')) and (req.if_match or req.if_none_match)
if slo_etag and slo_size and (req.method == 'HEAD' or is_conditional):
# Since we have length and etag, we can respond immediately
for i, (header, _value) in enumerate(self._response_headers):
lheader = header.lower()
if lheader == 'etag':
self._response_headers[i] = (header, '"%s"' % slo_etag)
elif lheader == 'content-length' and not is_conditional:
self._response_headers[i] = (header, slo_size)
start_response(self._response_status, self._response_headers,
self._response_exc_info)
return resp_iter
if self._need_to_refetch_manifest(req):
req.environ['swift.non_client_disconnect'] = True
close_if_possible(resp_iter)
del req.environ['swift.non_client_disconnect']
get_req = make_subrequest(
req.environ,
method='GET',
headers={'x-auth-token': req.headers.get('x-auth-token')},
agent='%(orig)s SLO MultipartGET',
swift_source='SLO')
resp_iter = self._app_call(get_req.environ)
# Any Content-Range from a manifest is almost certainly wrong for the
# full large object.
resp_headers = [(h, v) for h, v in self._response_headers
if not h.lower() == 'content-range']
response = self.get_or_head_response(req, resp_headers, resp_iter)
return response(req.environ, start_response)
def handle_slo_get_or_head(self, req, start_response):
"""
Takes a request and a start_response callable and does the normal WSGI
thing with them. Returns an iterator suitable for sending up the WSGI
chain.
:param req: swob.Request object; is a GET or HEAD request aimed at
what may be a static large object manifest (or may not).
:param start_response: WSGI start_response callable
"""
if req.params.get('multipart-manifest') != 'get':
# If this object is an SLO manifest, we may have saved off the
# large object etag during the original PUT. Send an
# X-Backend-Etag-Is-At header so that, if the SLO etag *was*
# saved, we can trust the object-server to respond appropriately
# to If-Match/If-None-Match requests.
update_etag_is_at_header(req, SYSMETA_SLO_ETAG)
resp_iter = self._app_call(req.environ)
# make sure this response is for a static large object manifest
slo_marker = slo_etag = slo_size = None
for header, value in self._response_headers:
header = header.lower()
if header == SYSMETA_SLO_ETAG:
slo_etag = value
elif header == SYSMETA_SLO_SIZE:
slo_size = value
elif (header == 'x-static-large-object' and
config_true_value(value)):
slo_marker = value
if slo_marker and slo_etag and slo_size:
break
if not slo_marker:
# Not a static large object manifest. Just pass it through.
start_response(self._response_status,
self._response_headers,
self._response_exc_info)
return resp_iter
# Handle pass-through request for the manifest itself
if req.params.get('multipart-manifest') == 'get':
if req.params.get('format') == 'raw':
resp_iter = self.convert_segment_listing(
self._response_headers, resp_iter)
else:
new_headers = []
for header, value in self._response_headers:
if header.lower() == 'content-type':
new_headers.append(('Content-Type',
'application/json; charset=utf-8'))
else:
new_headers.append((header, value))
self._response_headers = new_headers
start_response(self._response_status,
self._response_headers,
self._response_exc_info)
return resp_iter
is_conditional = self._response_status.startswith(('304', '412')) and (
req.if_match or req.if_none_match)
if slo_etag and slo_size and (
req.method == 'HEAD' or is_conditional):
# Since we have length and etag, we can respond immediately
for i, (header, _value) in enumerate(self._response_headers):
lheader = header.lower()
if lheader == 'etag':
self._response_headers[i] = (header, '"%s"' % slo_etag)
elif lheader == 'content-length' and not is_conditional:
self._response_headers[i] = (header, slo_size)
start_response(self._response_status,
self._response_headers,
self._response_exc_info)
return resp_iter
if self._need_to_refetch_manifest(req):
req.environ['swift.non_client_disconnect'] = True
close_if_possible(resp_iter)
del req.environ['swift.non_client_disconnect']
get_req = make_subrequest(
req.environ, method='GET',
headers={'x-auth-token': req.headers.get('x-auth-token')},
agent='%(orig)s SLO MultipartGET', swift_source='SLO')
resp_iter = self._app_call(get_req.environ)
# Any Content-Range from a manifest is almost certainly wrong for the
# full large object.
resp_headers = [(h, v) for h, v in self._response_headers
if not h.lower() == 'content-range']
response = self.get_or_head_response(
req, resp_headers, resp_iter)
return response(req.environ, start_response)
def GETorHEAD(self, req):
"""
Handled GET or HEAD request on a part of a multipart object.
"""
part_number = self.parse_part_number(req)
had_match = False
for match_header in ('if-match', 'if-none-match'):
if match_header not in req.headers:
continue
had_match = True
for value in list_from_csv(req.headers[match_header]):
if value.startswith('"') and value.endswith('"'):
value = value[1:-1]
if value.endswith('-N'):
# Deal with fake S3-like etags for SLOs uploaded via Swift
req.headers[match_header] += ', ' + value[:-2]
if had_match:
# Update where to look
update_etag_is_at_header(req, sysmeta_header('object', 'etag'))
# Get the list of parts. Must be raw to get all response headers.
slo_resp = req.get_response(self.app,
'GET',
req.container_name,
req.object_name,
query={
'multipart-manifest': 'get',
'format': 'raw'
})
# Check if the object is really a SLO. If not, and user asked
# for the first part, do a regular request.
if 'X-Static-Large-Object' not in slo_resp.sw_headers:
if part_number == 1:
if slo_resp.is_success and req.method == 'HEAD':
# Clear body
slo_resp.body = ''
return slo_resp
else:
close_if_possible(slo_resp.app_iter)
raise InvalidRange()
# Locate the part
slo = json.loads(slo_resp.body)
try:
part = slo[part_number - 1]
except IndexError:
raise InvalidRange()
# Redirect the request on the part
_, req.container_name, req.object_name = part['path'].split('/', 2)
# XXX enforce container_name and object_name to be <str>
# or it will rise issues in swift3/requests when merging both
req.container_name = req.container_name.encode('utf-8')
req.object_name = req.object_name.encode('utf8')
# The etag check was performed with the manifest
if had_match:
for match_header in ('if-match', 'if-none-match'):
req.headers.pop(match_header, None)
resp = req.get_response(self.app)
# Replace status
slo_resp.status = resp.status
# Replace body
slo_resp.app_iter = resp.app_iter
# Update with the size of the part
slo_resp.headers['Content-Length'] = \
resp.headers.get('Content-Length', 0)
slo_resp.sw_headers['Content-Length'] = \
slo_resp.headers['Content-Length']
# Add the number of parts in this object
slo_resp.headers['X-Amz-Mp-Parts-Count'] = len(slo)
return slo_resp
