def handle_put(self, req):
"""
Handle put request when it contains X-Symlink-Target header.
Symlink headers are validated and moved to sysmeta namespace.
:param req: HTTP PUT object request
:returns: Response Iterator
"""
if req.content_length is None:
has_body = (req.body_file.read(1) != b'')
else:
has_body = (req.content_length != 0)
if has_body:
raise HTTPBadRequest(
body='Symlink requests require a zero byte body',
request=req,
content_type='text/plain')
symlink_target_path, etag = _validate_and_prep_request_headers(req)
if etag:
self._validate_etag_and_update_sysmeta(
req, symlink_target_path, etag)
# N.B. TGT_ETAG_SYMLINK_HDR was converted as part of verifying it
symlink_usermeta_to_sysmeta(req.headers)
# Store info in container update that this object is a symlink.
# We have a design decision to use etag space to store symlink info for
# object listing because it's immutable unless the object is
# overwritten. This may impact the downgrade scenario that the symlink
# info can appear as the suffix in the hash value of object
# listing result for clients.
# To create override etag easily, we have a constraint that the symlink
# must be 0 byte so we can add etag of the empty string + symlink info
# here, simply (if no other override etag was provided). Note that this
# override etag may be encrypted in the container db by encryption
# middleware.
etag_override = [
req.headers.get(get_container_update_override_key('etag'),
MD5_OF_EMPTY_STRING),
'symlink_target=%s' % req.headers[TGT_OBJ_SYSMETA_SYMLINK_HDR]
]
if TGT_ACCT_SYSMETA_SYMLINK_HDR in req.headers:
etag_override.append(
'symlink_target_account=%s' %
req.headers[TGT_ACCT_SYSMETA_SYMLINK_HDR])
if TGT_ETAG_SYSMETA_SYMLINK_HDR in req.headers:
# if _validate_etag_and_update_sysmeta or a middleware sets
# TGT_ETAG_SYSMETA_SYMLINK_HDR then they need to also set
# TGT_BYTES_SYSMETA_SYMLINK_HDR. If they forget, they get a
# KeyError traceback and client gets a ServerError
etag_override.extend([
'symlink_target_etag=%s' %
req.headers[TGT_ETAG_SYSMETA_SYMLINK_HDR],
'symlink_target_bytes=%s' %
req.headers[TGT_BYTES_SYSMETA_SYMLINK_HDR],
])
req.headers[get_container_update_override_key('etag')] = \
'; '.join(etag_override)
return self._app_call(req.environ)
def decrypt_resp_headers(self, put_keys, post_keys):
"""
Find encrypted headers and replace with the decrypted versions.
:param put_keys: a dict of decryption keys used for object PUT.
:param post_keys: a dict of decryption keys used for object POST.
:return: A list of headers with any encrypted headers replaced by their
decrypted values.
:raises HTTPInternalServerError: if any error occurs while decrypting
headers
:raises HTTPForbidden: if the decryption key is invalid
"""
mod_hdr_pairs = []
if put_keys:
# Decrypt plaintext etag and place in Etag header for client
# response
etag_header = 'X-Object-Sysmeta-Crypto-Etag'
encrypted_etag = self._response_header_value(etag_header)
decrypted_etag = None
if encrypted_etag and 'object' in put_keys:
decrypted_etag = self._decrypt_header(etag_header,
encrypted_etag,
put_keys['object'],
required=True)
mod_hdr_pairs.append(('Etag', decrypted_etag))
etag_header = get_container_update_override_key('etag')
encrypted_etag = self._response_header_value(etag_header)
if encrypted_etag and 'container' in put_keys:
dcrypt_etag_override = self._decrypt_header(
etag_header, encrypted_etag, put_keys['container'])
if decrypted_etag and dcrypt_etag_override != decrypted_etag:
self.app.logger.debug('Failed ETag verification')
raise HTTPForbidden('Invalid key')
mod_hdr_pairs.append((etag_header, dcrypt_etag_override))
# The real swift saves the cyphered ETag in the 'ETag' field,
# whereas we store the ETag of the cyphered object.
# The ETag of the cyphered object is of no use for previous
# middlewares, so we replace it with the plaintext ETag.
mod_hdr_pairs.append(('ETag', dcrypt_etag_override))
# Decrypt all user metadata. Encrypted user metadata values are stored
# in the x-object-transient-sysmeta-crypto-meta- namespace. Those are
# decrypted and moved back to the x-object-meta- namespace. Prior to
# decryption, the response should have no x-object-meta- headers, but
# if it does then they will be overwritten by any decrypted headers
# that map to the same x-object-meta- header names i.e. decrypted
# headers win over unexpected, unencrypted headers.
try:
if post_keys:
mod_hdr_pairs.extend(self.decrypt_user_metadata(post_keys))
mod_hdr_names = {h.lower() for h, v in mod_hdr_pairs}
mod_hdr_pairs.extend([(h, v) for h, v in self._response_headers
if h.lower() not in mod_hdr_names])
except KeyError:
self.app.logger.debug('Not able to decrypt user metadata')
return mod_hdr_pairs
def _validate_etag_and_update_sysmeta(self, req, symlink_target_path,
etag):
if req.environ.get('swift.symlink_override'):
req.headers[TGT_ETAG_SYSMETA_SYMLINK_HDR] = etag
req.headers[TGT_BYTES_SYSMETA_SYMLINK_HDR] = \
req.headers[TGT_BYTES_SYMLINK_HDR]
return
# next we'll make sure the E-Tag matches a real object
new_req = make_subrequest(req.environ,
path=wsgi_quote(symlink_target_path),
method='HEAD',
swift_source='SYM')
if req.allow_reserved_names:
new_req.headers['X-Backend-Allow-Reserved-Names'] = 'true'
self._last_target_path = symlink_target_path
resp = self._recursive_get_head(new_req,
target_etag=etag,
follow_softlinks=False)
if self._get_status_int() == HTTP_NOT_FOUND:
raise HTTPConflict(body='X-Symlink-Target does not exist',
request=req,
headers={
'Content-Type': 'text/plain',
'Content-Location': self._last_target_path
})
if not is_success(self._get_status_int()):
drain_and_close(resp)
raise status_map[self._get_status_int()](request=req)
response_headers = HeaderKeyDict(self._response_headers)
# carry forward any etag update params (e.g. "slo_etag"), we'll append
# symlink_target_* params to this header after this method returns
override_header = get_container_update_override_key('etag')
if override_header in response_headers and \
override_header not in req.headers:
sep, params = response_headers[override_header].partition(';')[1:]
req.headers[override_header] = MD5_OF_EMPTY_STRING + sep + params
# It's troublesome that there's so much leakage with SLO
if 'X-Object-Sysmeta-Slo-Etag' in response_headers and \
override_header not in req.headers:
req.headers[override_header] = '%s; slo_etag=%s' % (
MD5_OF_EMPTY_STRING,
response_headers['X-Object-Sysmeta-Slo-Etag'])
req.headers[TGT_BYTES_SYSMETA_SYMLINK_HDR] = (
response_headers.get('x-object-sysmeta-slo-size')
or response_headers['Content-Length'])
req.headers[TGT_ETAG_SYSMETA_SYMLINK_HDR] = etag
if not req.headers.get('Content-Type'):
req.headers['Content-Type'] = response_headers['Content-Type']
def decrypt_resp_headers(self, put_keys, post_keys):
"""
Find encrypted headers and replace with the decrypted versions.
:param put_keys: a dict of decryption keys used for object PUT.
:param post_keys: a dict of decryption keys used for object POST.
:return: A list of headers with any encrypted headers replaced by their
decrypted values.
:raises HTTPInternalServerError: if any error occurs while decrypting
headers
"""
mod_hdr_pairs = []
if put_keys:
# Decrypt plaintext etag and place in Etag header for client
# response
etag_header = 'X-Object-Sysmeta-Crypto-Etag'
encrypted_etag = self._response_header_value(etag_header)
if encrypted_etag:
decrypted_etag = self._decrypt_header(
etag_header, encrypted_etag, put_keys['object'],
required=True)
mod_hdr_pairs.append(('Etag', decrypted_etag))
etag_header = get_container_update_override_key('etag')
encrypted_etag = self._response_header_value(etag_header)
if encrypted_etag:
decrypted_etag = self._decrypt_header(
etag_header, encrypted_etag, put_keys['container'])
mod_hdr_pairs.append((etag_header, decrypted_etag))
# Decrypt all user metadata. Encrypted user metadata values are stored
# in the x-object-transient-sysmeta-crypto-meta- namespace. Those are
# decrypted and moved back to the x-object-meta- namespace. Prior to
# decryption, the response should have no x-object-meta- headers, but
# if it does then they will be overwritten by any decrypted headers
# that map to the same x-object-meta- header names i.e. decrypted
# headers win over unexpected, unencrypted headers.
if post_keys:
mod_hdr_pairs.extend(self.decrypt_user_metadata(post_keys))
mod_hdr_names = {h.lower() for h, v in mod_hdr_pairs}
mod_hdr_pairs.extend([(h, v) for h, v in self._response_headers
if h.lower() not in mod_hdr_names])
return mod_hdr_pairs
def handle_put(self, req):
"""
Handle put request when it contains X-Symlink-Target header.
Symlink headers are validated and moved to sysmeta namespace.
:param req: HTTP PUT object request
:returns: Response Iterator
"""
if req.content_length is None:
has_body = (req.body_file.read(1) != b'')
else:
has_body = (req.content_length != 0)
if has_body:
raise HTTPBadRequest(
body='Symlink requests require a zero byte body',
request=req,
content_type='text/plain')
_check_symlink_header(req)
symlink_usermeta_to_sysmeta(req.headers)
# Store info in container update that this object is a symlink.
# We have a design decision to use etag space to store symlink info for
# object listing because it's immutable unless the object is
# overwritten. This may impact the downgrade scenario that the symlink
# info can appear as the suffix in the hash value of object
# listing result for clients.
# To create override etag easily, we have a constraint that the symlink
# must be 0 byte so we can add etag of the empty string + symlink info
# here, simply. Note that this override etag may be encrypted in the
# container db by encryption middleware.
etag_override = [
MD5_OF_EMPTY_STRING,
'symlink_target=%s' % req.headers[TGT_OBJ_SYSMETA_SYMLINK_HDR]
]
if TGT_ACCT_SYSMETA_SYMLINK_HDR in req.headers:
etag_override.append('symlink_target_account=%s' %
req.headers[TGT_ACCT_SYSMETA_SYMLINK_HDR])
req.headers[get_container_update_override_key('etag')] = \
'; '.join(etag_override)
return self._app_call(req.environ)
def POST(self, req):
"""
Handles Complete Multipart Upload.
"""
upload_id = req.params['uploadId']
resp = _get_upload_info(req, self.app, upload_id)
headers = {
'Accept': 'application/json',
sysmeta_header('object', 'upload-id'): upload_id
}
for key, val in resp.headers.items():
_key = key.lower()
if _key.startswith('x-amz-meta-'):
headers['x-object-meta-' + _key[11:]] = val
hct_header = sysmeta_header('object', 'has-content-type')
if resp.sysmeta_headers.get(hct_header) == 'yes':
content_type = resp.sysmeta_headers.get(
sysmeta_header('object', 'content-type'))
elif hct_header in resp.sysmeta_headers:
# has-content-type is present but false, so no content type was
# set on initial upload. In that case, we won't set one on our
# PUT request. Swift will end up guessing one based on the
# object name.
content_type = None
else:
content_type = resp.headers.get('Content-Type')
if content_type:
headers['Content-Type'] = content_type
container = req.container_name + MULTIUPLOAD_SUFFIX
s3_etag_hasher = md5(usedforsecurity=False)
manifest = []
previous_number = 0
try:
xml = req.xml(MAX_COMPLETE_UPLOAD_BODY_SIZE)
if not xml:
raise InvalidRequest(msg='You must specify at least one part')
if 'content-md5' in req.headers:
# If an MD5 was provided, we need to verify it.
# Note that S3Request already took care of translating to ETag
if req.headers['etag'] != md5(
xml, usedforsecurity=False).hexdigest():
raise BadDigest(content_md5=req.headers['content-md5'])
# We're only interested in the body here, in the
# multipart-upload controller -- *don't* let it get
# plumbed down to the object-server
del req.headers['etag']
complete_elem = fromstring(xml, 'CompleteMultipartUpload',
self.logger)
for part_elem in complete_elem.iterchildren('Part'):
part_number = int(part_elem.find('./PartNumber').text)
if part_number <= previous_number:
raise InvalidPartOrder(upload_id=upload_id)
previous_number = part_number
etag = normalize_etag(part_elem.find('./ETag').text)
if len(etag) != 32 or any(c not in '0123456789abcdef'
for c in etag):
raise InvalidPart(upload_id=upload_id,
part_number=part_number)
manifest.append({
'path':
'/%s/%s/%s/%d' %
(wsgi_to_str(container), wsgi_to_str(
req.object_name), upload_id, part_number),
'etag':
etag
})
s3_etag_hasher.update(binascii.a2b_hex(etag))
except (XMLSyntaxError, DocumentInvalid):
# NB: our schema definitions catch uploads with no parts here
raise MalformedXML()
except ErrorResponse:
raise
except Exception as e:
self.logger.error(e)
raise
s3_etag = '%s-%d' % (s3_etag_hasher.hexdigest(), len(manifest))
s3_etag_header = sysmeta_header('object', 'etag')
if resp.sysmeta_headers.get(s3_etag_header) == s3_etag:
# This header should only already be present if the upload marker
# has been cleaned up and the current target uses the same
# upload-id; assuming the segments to use haven't changed, the work
# is already done
return HTTPOk(body=_make_complete_body(req, s3_etag, False),
content_type='application/xml')
headers[s3_etag_header] = s3_etag
# Leave base header value blank; SLO will populate
c_etag = '; s3_etag=%s' % s3_etag
headers[get_container_update_override_key('etag')] = c_etag
too_small_message = ('s3api requires that each segment be at least '
'%d bytes' % self.conf.min_segment_size)
def size_checker(manifest):
# Check the size of each segment except the last and make sure
# they are all more than the minimum upload chunk size.
# Note that we need to use the *internal* keys, since we're
# looking at the manifest that's about to be written.
return [(item['name'], too_small_message) for item in manifest[:-1]
if item and item['bytes'] < self.conf.min_segment_size]
req.environ['swift.callback.slo_manifest_hook'] = size_checker
start_time = time.time()
def response_iter():
# NB: XML requires that the XML declaration, if present, be at the
# very start of the document. Clients *will* call us out on not
# being valid XML if we pass through whitespace before it.
# Track whether we've sent anything yet so we can yield out that
# declaration *first*
yielded_anything = False
try:
try:
# TODO: add support for versioning
put_resp = req.get_response(self.app,
'PUT',
body=json.dumps(manifest),
query={
'multipart-manifest':
'put',
'heartbeat': 'on'
},
headers=headers)
if put_resp.status_int == 202:
body = []
put_resp.fix_conditional_response()
for chunk in put_resp.response_iter:
if not chunk.strip():
if time.time() - start_time < 10:
# Include some grace period to keep
# ceph-s3tests happy
continue
if not yielded_anything:
yield (b'<?xml version="1.0" '
b'encoding="UTF-8"?>\n')
yielded_anything = True
yield chunk
continue
body.append(chunk)
body = json.loads(b''.join(body))
if body['Response Status'] != '201 Created':
for seg, err in body['Errors']:
if err == too_small_message:
raise EntityTooSmall()
elif err in ('Etag Mismatch', '404 Not Found'):
raise InvalidPart(upload_id=upload_id)
raise InvalidRequest(
status=body['Response Status'],
msg='\n'.join(': '.join(err)
for err in body['Errors']))
except BadSwiftRequest as e:
msg = str(e)
if too_small_message in msg:
raise EntityTooSmall(msg)
elif ', Etag Mismatch' in msg:
raise InvalidPart(upload_id=upload_id)
elif ', 404 Not Found' in msg:
raise InvalidPart(upload_id=upload_id)
else:
raise
# clean up the multipart-upload record
obj = '%s/%s' % (req.object_name, upload_id)
try:
req.get_response(self.app, 'DELETE', container, obj)
except NoSuchKey:
# The important thing is that we wrote out a tombstone to
# make sure the marker got cleaned up. If it's already
# gone (e.g., because of concurrent completes or a retried
# complete), so much the better.
pass
yield _make_complete_body(req, s3_etag, yielded_anything)
except ErrorResponse as err_resp:
if yielded_anything:
err_resp.xml_declaration = False
yield b'\n'
else:
# Oh good, we can still change HTTP status code, too!
resp.status = err_resp.status
for chunk in err_resp({}, lambda *a: None):
yield chunk
resp = HTTPOk() # assume we're good for now... but see above!
resp.app_iter = reiterate(response_iter())
resp.content_type = "application/xml"
return resp
def PUT(self, req):
"""
Handles Upload Part and Upload Part Copy.
"""
if 'uploadId' not in req.params:
raise InvalidArgument('ResourceType', 'partNumber',
'Unexpected query string parameter')
try:
part_number = int(req.params['partNumber'])
if part_number < 1 or self.conf.max_upload_part_num < part_number:
raise Exception()
except Exception:
err_msg = 'Part number must be an integer between 1 and %d,' \
' inclusive' % self.conf.max_upload_part_num
raise InvalidArgument('partNumber', req.params['partNumber'],
err_msg)
upload_id = req.params['uploadId']
_get_upload_info(req, self.app, upload_id)
req.container_name += MULTIUPLOAD_SUFFIX
req.object_name = '%s/%s/%d' % (req.object_name, upload_id,
part_number)
req_timestamp = S3Timestamp.now()
req.headers['X-Timestamp'] = req_timestamp.internal
source_resp = req.check_copy_source(self.app)
if 'X-Amz-Copy-Source' in req.headers and \
'X-Amz-Copy-Source-Range' in req.headers:
rng = req.headers['X-Amz-Copy-Source-Range']
header_valid = True
try:
rng_obj = Range(rng)
if len(rng_obj.ranges) != 1:
header_valid = False
except ValueError:
header_valid = False
if not header_valid:
err_msg = ('The x-amz-copy-source-range value must be of the '
'form bytes=first-last where first and last are '
'the zero-based offsets of the first and last '
'bytes to copy')
raise InvalidArgument('x-amz-source-range', rng, err_msg)
source_size = int(source_resp.headers['Content-Length'])
if not rng_obj.ranges_for_length(source_size):
err_msg = ('Range specified is not valid for source object '
'of size: %s' % source_size)
raise InvalidArgument('x-amz-source-range', rng, err_msg)
req.headers['Range'] = rng
del req.headers['X-Amz-Copy-Source-Range']
if 'X-Amz-Copy-Source' in req.headers:
# Clear some problematic headers that might be on the source
req.headers.update({
sysmeta_header('object', 'etag'): '',
'X-Object-Sysmeta-Swift3-Etag': '', # for legacy data
'X-Object-Sysmeta-Slo-Etag': '',
'X-Object-Sysmeta-Slo-Size': '',
get_container_update_override_key('etag'): '',
})
resp = req.get_response(self.app)
if 'X-Amz-Copy-Source' in req.headers:
resp.append_copy_resp_body(req.controller_name,
req_timestamp.s3xmlformat)
resp.status = 200
return resp
def POST(self, req):
"""
Handles Complete Multipart Upload.
"""
upload_id = req.params['uploadId']
resp = _get_upload_info(req, self.app, upload_id)
headers = {'Accept': 'application/json'}
for key, val in resp.headers.items():
_key = key.lower()
if _key.startswith('x-amz-meta-'):
headers['x-object-meta-' + _key[11:]] = val
hct_header = sysmeta_header('object', 'has-content-type')
if resp.sysmeta_headers.get(hct_header) == 'yes':
content_type = resp.sysmeta_headers.get(
sysmeta_header('object', 'content-type'))
elif hct_header in resp.sysmeta_headers:
# has-content-type is present but false, so no content type was
# set on initial upload. In that case, we won't set one on our
# PUT request. Swift will end up guessing one based on the
# object name.
content_type = None
else:
content_type = resp.headers.get('Content-Type')
if content_type:
headers['Content-Type'] = content_type
container = req.container_name + MULTIUPLOAD_SUFFIX
s3_etag_hasher = md5()
manifest = []
previous_number = 0
try:
xml = req.xml(MAX_COMPLETE_UPLOAD_BODY_SIZE)
if not xml:
raise InvalidRequest(msg='You must specify at least one part')
if 'content-md5' in req.headers:
# If an MD5 was provided, we need to verify it.
# Note that S3Request already took care of translating to ETag
if req.headers['etag'] != md5(xml).hexdigest():
raise BadDigest(content_md5=req.headers['content-md5'])
# We're only interested in the body here, in the
# multipart-upload controller -- *don't* let it get
# plumbed down to the object-server
del req.headers['etag']
complete_elem = fromstring(xml, 'CompleteMultipartUpload',
self.logger)
for part_elem in complete_elem.iterchildren('Part'):
part_number = int(part_elem.find('./PartNumber').text)
if part_number <= previous_number:
raise InvalidPartOrder(upload_id=upload_id)
previous_number = part_number
etag = normalize_etag(part_elem.find('./ETag').text)
if len(etag) != 32 or any(c not in '0123456789abcdef'
for c in etag):
raise InvalidPart(upload_id=upload_id,
part_number=part_number)
manifest.append({
'path':
'/%s/%s/%s/%d' %
(container, req.object_name, upload_id, part_number),
'etag':
etag
})
s3_etag_hasher.update(binascii.a2b_hex(etag))
except (XMLSyntaxError, DocumentInvalid):
# NB: our schema definitions catch uploads with no parts here
raise MalformedXML()
except ErrorResponse:
raise
except Exception as e:
self.logger.error(e)
raise
s3_etag = '%s-%d' % (s3_etag_hasher.hexdigest(), len(manifest))
headers[sysmeta_header('object', 'etag')] = s3_etag
# Leave base header value blank; SLO will populate
c_etag = '; s3_etag=%s' % s3_etag
headers[get_container_update_override_key('etag')] = c_etag
too_small_message = ('s3api requires that each segment be at least '
'%d bytes' % self.conf.min_segment_size)
def size_checker(manifest):
# Check the size of each segment except the last and make sure
# they are all more than the minimum upload chunk size.
# Note that we need to use the *internal* keys, since we're
# looking at the manifest that's about to be written.
return [(item['name'], too_small_message) for item in manifest[:-1]
if item and item['bytes'] < self.conf.min_segment_size]
req.environ['swift.callback.slo_manifest_hook'] = size_checker
start_time = time.time()
def response_iter():
# NB: XML requires that the XML declaration, if present, be at the
# very start of the document. Clients *will* call us out on not
# being valid XML if we pass through whitespace before it.
# Track whether we've sent anything yet so we can yield out that
# declaration *first*
yielded_anything = False
try:
try:
# TODO: add support for versioning
put_resp = req.get_response(self.app,
'PUT',
body=json.dumps(manifest),
query={
'multipart-manifest':
'put',
'heartbeat': 'on'
},
headers=headers)
if put_resp.status_int == 202:
body = []
put_resp.fix_conditional_response()
for chunk in put_resp.response_iter:
if not chunk.strip():
if time.time() - start_time < 10:
# Include some grace period to keep
# ceph-s3tests happy
continue
if not yielded_anything:
yield (b'<?xml version="1.0" '
b'encoding="UTF-8"?>\n')
yielded_anything = True
yield chunk
continue
body.append(chunk)
body = json.loads(b''.join(body))
if body['Response Status'] != '201 Created':
for seg, err in body['Errors']:
if err == too_small_message:
raise EntityTooSmall()
elif err in ('Etag Mismatch', '404 Not Found'):
raise InvalidPart(upload_id=upload_id)
raise InvalidRequest(
status=body['Response Status'],
msg='\n'.join(': '.join(err)
for err in body['Errors']))
except BadSwiftRequest as e:
msg = str(e)
if too_small_message in msg:
raise EntityTooSmall(msg)
elif ', Etag Mismatch' in msg:
raise InvalidPart(upload_id=upload_id)
elif ', 404 Not Found' in msg:
raise InvalidPart(upload_id=upload_id)
else:
raise
# clean up the multipart-upload record
obj = '%s/%s' % (req.object_name, upload_id)
try:
req.get_response(self.app, 'DELETE', container, obj)
except NoSuchKey:
# We know that this existed long enough for us to HEAD
pass
result_elem = Element('CompleteMultipartUploadResult')
# NOTE: boto with sig v4 appends port to HTTP_HOST value at
# the request header when the port is non default value and it
# makes req.host_url like as http://localhost:8080:8080/path
# that obviously invalid. Probably it should be resolved at
# swift.common.swob though, tentatively we are parsing and
# reconstructing the correct host_url info here.
# in detail, https://github.com/boto/boto/pull/3513
parsed_url = urlparse(req.host_url)
host_url = '%s://%s' % (parsed_url.scheme, parsed_url.hostname)
# Why are we doing our own port parsing? Because py3 decided
# to start raising ValueErrors on access after parsing such
# an invalid port
netloc = parsed_url.netloc.split('@')[-1].split(']')[-1]
if ':' in netloc:
port = netloc.split(':', 2)[1]
host_url += ':%s' % port
SubElement(result_elem, 'Location').text = host_url + req.path
SubElement(result_elem, 'Bucket').text = req.container_name
SubElement(result_elem, 'Key').text = req.object_name
SubElement(result_elem, 'ETag').text = '"%s"' % s3_etag
resp.headers.pop('ETag', None)
if yielded_anything:
yield b'\n'
yield tostring(result_elem,
xml_declaration=not yielded_anything)
except ErrorResponse as err_resp:
if yielded_anything:
err_resp.xml_declaration = False
yield b'\n'
else:
# Oh good, we can still change HTTP status code, too!
resp.status = err_resp.status
for chunk in err_resp({}, lambda *a: None):
yield chunk
resp = HTTPOk() # assume we're good for now... but see above!
resp.app_iter = reiterate(response_iter())
resp.content_type = "application/xml"
return resp
def install_footers_callback(self, req):
# the proxy controller will call back for footer metadata after
# body has been sent
inner_callback = req.environ.get('swift.callback.update_footers')
# remove any Etag from headers, it won't be valid for ciphertext and
# we'll send the ciphertext Etag later in footer metadata
client_etag = req.headers.pop('etag', None)
override_header = get_container_update_override_key('etag')
container_listing_etag_header = req.headers.get(override_header)
def footers_callback(footers):
if inner_callback:
# pass on footers dict to any other callback that was
# registered before this one. It may override any footers that
# were set.
inner_callback(footers)
plaintext_etag = None
if self.body_crypto_ctxt:
plaintext_etag = self.plaintext_md5.hexdigest()
# If client (or other middleware) supplied etag, then validate
# against plaintext etag
etag_to_check = footers.get('Etag') or client_etag
if (etag_to_check is not None
and plaintext_etag != etag_to_check):
raise HTTPUnprocessableEntity(request=Request(self.env))
# override any previous notion of etag with the ciphertext etag
footers['Etag'] = self.ciphertext_md5.hexdigest()
# Encrypt the plaintext etag using the object key and persist
# as sysmeta along with the crypto parameters that were used.
encrypted_etag, etag_crypto_meta = encrypt_header_val(
self.crypto, plaintext_etag, self.keys['object'])
footers['X-Object-Sysmeta-Crypto-Etag'] = \
append_crypto_meta(encrypted_etag, etag_crypto_meta)
footers['X-Object-Sysmeta-Crypto-Body-Meta'] = \
dump_crypto_meta(self.body_crypto_meta)
# Also add an HMAC of the etag for use when evaluating
# conditional requests
footers['X-Object-Sysmeta-Crypto-Etag-Mac'] = _hmac_etag(
self.keys['object'], plaintext_etag)
else:
# No data was read from body, nothing was encrypted, so don't
# set any crypto sysmeta for the body, but do re-instate any
# etag provided in inbound request if other middleware has not
# already set a value.
if client_etag is not None:
footers.setdefault('Etag', client_etag)
# When deciding on the etag that should appear in container
# listings, look for:
# * override in the footer, otherwise
# * override in the header, and finally
# * MD5 of the plaintext received
# This may be None if no override was set and no data was read. An
# override value of '' will be passed on.
container_listing_etag = footers.get(
override_header, container_listing_etag_header)
if container_listing_etag is None:
container_listing_etag = plaintext_etag
if (container_listing_etag
and (container_listing_etag != MD5_OF_EMPTY_STRING
or plaintext_etag)):
# Encrypt the container-listing etag using the container key
# and a random IV, and use it to override the container update
# value, with the crypto parameters appended. We use the
# container key here so that only that key is required to
# decrypt all etag values in a container listing when handling
# a container GET request. Don't encrypt an EMPTY_ETAG
# unless there actually was some body content, in which case
# the container-listing etag is possibly conveying some
# non-obvious information.
val, crypto_meta = encrypt_header_val(self.crypto,
container_listing_etag,
self.keys['container'])
crypto_meta['key_id'] = self.keys['id']
footers[override_header] = \
append_crypto_meta(val, crypto_meta)
# else: no override was set and no data was read
req.environ['swift.callback.update_footers'] = footers_callback
