def PUT(self, req):
"""
Handles PUT Bucket acl and PUT Object acl.
"""
log_s3api_command(req, 'put-acl')
if req.is_object_request:
# Handle Object ACL
raise S3NotImplemented()
else:
# Handle Bucket ACL
xml = req.xml(MAX_ACL_BODY_SIZE)
if all(['HTTP_X_AMZ_ACL' in req.environ, xml]):
# S3 doesn't allow to give ACL with both ACL header and body.
raise UnexpectedContent()
elif not any(['HTTP_X_AMZ_ACL' in req.environ, xml]):
# Both canned ACL header and xml body are missing
raise MissingSecurityHeader(missing_header_name='x-amz-acl')
else:
# correct ACL exists in the request
if xml:
# We very likely have an XML-based ACL request.
# let's try to translate to the request header
try:
translated_acl = swift_acl_translate(xml, xml=True)
except ACLError:
raise MalformedACLError()
for header, acl in translated_acl:
req.headers[header] = acl
resp = req.get_response(self.app, 'POST')
resp.status = HTTP_OK
resp.headers.update({'Location': req.container_name})
return resp
def PUT(self, req):
"""
Handles PUT Bucket versioning.
"""
log_s3api_command(req, 'put-bucket-versioning')
xml = req.xml(MAX_PUT_VERSIONING_BODY_SIZE)
try:
elem = fromstring(xml, 'VersioningConfiguration')
status = elem.find('./Status').text
except (XMLSyntaxError, DocumentInvalid):
raise MalformedXML()
except Exception as e:
LOGGER.error(e)
raise
if status not in ['Enabled', 'Suspended']:
raise MalformedXML()
# Make sure the versions container exists
req.container_name += VERSIONING_SUFFIX
try:
req.get_container_info(self.app)
except NoSuchBucket:
req.get_response(self.app, 'PUT', req.container_name, '')
# Set up versioning
if status == 'Enabled':
req.headers['X-History-Location'] = req.container_name
else:
req.headers['X-Remove-History-Location'] = 'true'
# Set the container back to what it originally was
req.container_name = req.container_name[:-len(VERSIONING_SUFFIX)]
resp = req.get_response(self.app, 'POST')
return convert_response(req, resp, 204, HTTPOk)
def GET(self, req):
"""
Handles GET Bucket versioning.
"""
log_s3api_command(req, 'get-bucket-versioning')
info = req.get_container_info(self.app)
status = None
versions_container = info.get('sysmeta', {}).get('versions-location')
if versions_container:
status = 'Enabled'
else:
versions_container = ''.join(
[req.container_name, VERSIONING_SUFFIX])
try:
req.get_response(self.app,
'HEAD',
container=versions_container)
status = 'Suspended'
except NoSuchBucket:
pass
# Just report there is no versioning configured here.
elem = Element('VersioningConfiguration')
if status:
SubElement(elem, 'Status').text = status
body = tostring(elem)
return HTTPOk(body=body, content_type="text/plain")
def PUT(self, req):
"""
Handle PUT Bucket request
"""
log_s3api_command(req, 'create-bucket')
xml = req.xml(MAX_PUT_BUCKET_BODY_SIZE)
if xml:
# check location
try:
elem = fromstring(xml, 'CreateBucketConfiguration')
location = elem.find('./LocationConstraint').text
except (XMLSyntaxError, DocumentInvalid):
raise MalformedXML()
except Exception as e:
LOGGER.error(e)
raise
if location != CONF.location:
# Swift3 cannot support multiple regions currently.
raise InvalidLocationConstraint()
resp = req.get_response(self.app)
# create bucket+segments to avoid breaking
# functional tests for swift3
try:
cnt = req.container_name + MULTIUPLOAD_SUFFIX
req.get_response(self.app, 'PUT', cnt, '')
except BucketAlreadyExists:
pass
resp.status = HTTP_OK
resp.location = '/' + req.container_name
return resp
def POST(self, req):
"""
Handles Initiate Multipart Upload.
"""
log_s3api_command(req, 'create-multipart-upload')
# Create a unique S3 upload id from UUID to avoid duplicates.
upload_id = unique_id()
container = req.container_name + MULTIUPLOAD_SUFFIX
obj = '%s/%s' % (req.object_name, upload_id)
if HTTP_HEADER_TAGGING_KEY in req.headers:
tagging = convert_urlquery_to_xml(
req.headers.get(HTTP_HEADER_TAGGING_KEY))
req.headers[OBJECT_TAGGING_HEADER] = tagging
req.headers.pop('Etag', None)
req.headers.pop('Content-Md5', None)
req.environ['oio.ephemeral_object'] = True
req.get_response(self.app, 'PUT', container, obj, body='')
result_elem = Element('InitiateMultipartUploadResult')
SubElement(result_elem, 'Bucket').text = req.container_name
SubElement(result_elem, 'Key').text = req.object_name
SubElement(result_elem, 'UploadId').text = upload_id
body = tostring(result_elem)
return HTTPOk(body=body, content_type='application/xml')
def PUT(self, req):
"""
Handle PUT Object and PUT Object (Copy) request
"""
method = 'put-object'
# set X-Timestamp by swift3 to use at copy resp body
req_timestamp = S3Timestamp.now()
req.headers['X-Timestamp'] = req_timestamp.internal
if all(h in req.headers
for h in ('X-Amz-Copy-Source', 'X-Amz-Copy-Source-Range')):
raise InvalidArgument('x-amz-copy-source-range',
req.headers['X-Amz-Copy-Source-Range'],
'Illegal copy header')
if HTTP_HEADER_TAGGING_KEY in req.headers:
tagging = convert_urlquery_to_xml(
req.headers.pop(HTTP_HEADER_TAGGING_KEY))
req.headers[OBJECT_TAGGING_HEADER] = tagging
req.check_copy_source(self.app)
resp = req.get_response(self.app)
if 'X-Amz-Copy-Source' in req.headers:
method = 'copy-object'
resp.append_copy_resp_body(req.controller_name,
req_timestamp.s3xmlformat)
# delete object metadata from response
for key in list(resp.headers.keys()):
if key.startswith('x-amz-meta-'):
del resp.headers[key]
log_s3api_command(req, method)
resp.status = HTTP_OK
return resp
def DELETE(self, req):
"""
Handle DELETE Object request
"""
log_s3api_command(req, 'delete-object')
try:
query = req.gen_multipart_manifest_delete_query(self.app)
req.headers['Content-Type'] = None # Ignore client content-type
if req.params.get('versionId'):
resp = self._delete_version(req, query)
else:
ctinfo = req.get_container_info(self.app)
if ctinfo.get('sysmeta', {}).get('versions-mode') == 'history':
# If the object is a manifest, and versioning is enabled,
# we must not delete the parts!
resp = req.get_response(self.app)
else:
resp = req.get_response(self.app, query=query)
if query and resp.status_int == HTTP_OK:
for chunk in resp.app_iter:
pass # drain the bulk-deleter response
resp.status = HTTP_NO_CONTENT
resp.body = ''
except NoSuchKey:
# expect to raise NoSuchBucket when the bucket doesn't exist
req.get_container_info(self.app)
raise
return resp
def GET(self, req):
"""
Handles GET Bucket acl and GET Object acl.
"""
log_s3api_command(req, 'get-acl')
resp = req.get_response(self.app, method='HEAD')
return get_acl(req.user_id, resp.headers)
def DELETE(self, req): # pylint: disable=invalid-name
"""
Handles DELETE Bucket CORS.
"""
log_s3api_command(req, 'delete-bucket-cors')
req.headers[BUCKET_CORS_HEADER] = ''
resp = req.get_response(self.app, 'POST')
return convert_response(req, resp, 202, HTTPNoContent)
def DELETE(self, req):
"""
Handle DELETE Bucket request
"""
log_s3api_command(req, 'delete-bucket')
if CONF.allow_multipart_uploads:
self._delete_segments_bucket(req)
resp = req.get_response(self.app)
return resp
def HEAD(self, req):
"""
Handle HEAD Bucket (Get Metadata) request
"""
log_s3api_command(req, 'head-bucket')
resp = req.get_response(self.app)
return HTTPOk(headers=resp.headers)
def GET(self, req): # pylint: disable=invalid-name
"""
Handles GET Bucket CORS.
"""
log_s3api_command(req, 'get-bucket-cors')
resp = req.get_response(self.app, method='HEAD')
body = resp.sysmeta_headers.get(BUCKET_CORS_HEADER)
if not body:
raise NoSuchCORSConfiguration
return HTTPOk(body=body, content_type='application/xml')
def HEAD(self, req):
"""
Handle HEAD Object request
"""
log_s3api_command(req, 'head-object')
resp = self.GETorHEAD(req)
if 'range' in req.headers:
req_range = req.headers['range']
resp = self._gen_head_range_resp(req_range, resp)
return resp
def GET(self, req):
"""
Handle GET Service request
"""
log_s3api_command(req, 'list-buckets')
resp = req.get_response(self.app, query={'format': 'json'})
containers = json.loads(resp.body)
containers = filter(
lambda item: validate_bucket_name(item['name']), containers)
# we don't keep the creation time of a bucket (s3cmd doesn't
# work without that) so we use something bogus.
elem = Element('ListAllMyBucketsResult')
owner = SubElement(elem, 'Owner')
SubElement(owner, 'ID').text = req.user_id
SubElement(owner, 'DisplayName').text = req.user_id
buckets = SubElement(elem, 'Buckets')
for c in containers:
if 'last_modified' in c:
ts = last_modified_date_to_timestamp(c['last_modified'])
creation_date = S3Timestamp(ts).s3xmlformat
else:
creation_date = '2009-02-03T16:45:09.000Z'
if CONF.s3_acl and CONF.check_bucket_owner:
try:
cname = c['name'].encode('utf-8')
c_resp = req.get_response(self.app, 'HEAD', cname)
if 'X-Timestamp' in c_resp.sw_headers:
creation_date = S3Timestamp(
c_resp.sw_headers['X-Timestamp']).s3xmlformat
except AccessDenied:
continue
except NoSuchBucket:
continue
bucket = SubElement(buckets, 'Bucket')
SubElement(bucket, 'Name').text = c['name']
SubElement(bucket, 'CreationDate').text = creation_date
body = tostring(elem)
return HTTPOk(content_type='application/xml', body=body)
def DELETE(self, req):
"""
Handles Abort Multipart Upload.
"""
log_s3api_command(req, 'abort-multipart-upload')
upload_id = req.params['uploadId']
_check_upload_info(req, self.app, upload_id)
container = req.container_name + MULTIUPLOAD_SUFFIX
# First check to see if this multi-part upload was already
# completed. Look in the primary container, if the object exists,
# then it was completed and we return an error here.
obj = '%s/%s' % (req.object_name, upload_id)
req.environ['oio.ephemeral_object'] = True
try:
req.get_response(self.app, container=container, obj=obj)
finally:
req.environ['oio.ephemeral_object'] = False
# The completed object was not found so this
# must be a multipart upload abort.
# We must delete any uploaded segments for this UploadID and then
# delete the object in the main container as well
query = {
'format': 'json',
'prefix': '%s/%s/' % (req.object_name, upload_id),
'delimiter': '/',
}
# Force the master to be sure to fetch all uploaded parts
req.environ.setdefault('oio.query', {})
req.environ['oio.query']['force_master'] = True
resp = req.get_response(self.app, 'GET', container, '', query=query)
# Iterate over the segment objects and delete them individually
objects = json.loads(resp.body)
for o in objects:
container = req.container_name + MULTIUPLOAD_SUFFIX
obj = o['name'].encode('utf-8')
req.get_response(self.app, container=container, obj=obj)
return HTTPNoContent()
def OPTIONS(self, req):
log_s3api_command(req, 'options')
origin = req.headers.get('Origin')
if not origin:
raise CORSOriginMissing()
method = req.headers.get('Access-Control-Request-Method')
if method not in CORS_ALLOWED_HTTP_METHOD:
raise CORSInvalidAccessControlRequest(method=method)
rule = get_cors(self.app, req, method, origin)
# FIXME(mbo): we should raise also NoSuchCORSConfiguration
if rule is None:
raise CORSForbidden(method)
resp = HTTPOk(body=None)
del resp.headers['Content-Type']
return cors_fill_headers(req, resp, rule)
def PUT(self, req): # pylint: disable=invalid-name
"""
Handles PUT Bucket CORS.
"""
log_s3api_command(req, 'put-bucket-cors')
xml = req.xml(MAX_CORS_BODY_SIZE)
try:
data = fromstring(xml, "CorsConfiguration")
except (XMLSyntaxError, DocumentInvalid):
raise MalformedXML()
except Exception as exc:
LOGGER.error(exc)
raise
# forbid wildcard for ExposeHeader
check_cors_rule(data)
req.headers[BUCKET_CORS_HEADER] = xml
resp = req.get_response(self.app, 'POST')
return convert_response(req, resp, 204, HTTPOk)
def DELETE(self, req):
"""
Handle DELETE Bucket request
"""
log_s3api_command(req, 'delete-bucket')
try:
resp = super(UniqueBucketController, self).DELETE(req)
except NoSuchBucket:
ct_owner = req.bucket_db.get_owner(req.container_name)
if ct_owner == req.account:
# The bucket used to be ours, but for some reason
# it has not been released.
req.bucket_db.release(req.container_name)
raise
if resp.is_success:
# Container deletion succeeded, reset owner.
req.bucket_db.release(req.container_name)
return resp
def PUT(self, req):
"""
Handle PUT Bucket request
"""
log_s3api_command(req, 'create-bucket')
# We are about to create a container, reserve its name.
can_create = req.bucket_db.reserve(req.container_name, req.account)
if not can_create:
ct_owner = req.bucket_db.get_owner(req.container_name)
if ct_owner == req.account:
raise BucketAlreadyOwnedByYou(req.container_name)
raise BucketAlreadyExists(req.container_name)
try:
resp = super(UniqueBucketController, self).PUT(req)
except Exception:
# Container creation failed, remove reservation.
req.bucket_db.release(req.container_name)
raise
# Container creation succeeded, confirm reservation.
req.bucket_db.set_owner(req.container_name, req.account)
return resp
def PUT(self, req):
"""
Handles Upload Part and Upload Part Copy.
"""
if 'uploadId' not in req.params:
raise InvalidArgument('ResourceType', 'partNumber',
'Unexpected query string parameter')
part_number = self.parse_part_number(req)
upload_id = req.params['uploadId']
_check_upload_info(req, self.app, upload_id)
req.container_name += MULTIUPLOAD_SUFFIX
req.object_name = '%s/%s/%d' % (req.object_name, upload_id,
part_number)
req_timestamp = S3Timestamp.now()
req.headers['X-Timestamp'] = req_timestamp.internal
source_resp = req.check_copy_source(self.app)
method = 'upload-part'
if 'X-Amz-Copy-Source' in req.headers and \
'X-Amz-Copy-Source-Range' in req.headers:
rng = req.headers['X-Amz-Copy-Source-Range']
method = 'upload-part-copy'
header_valid = True
try:
rng_obj = Range(rng)
if len(rng_obj.ranges) != 1:
header_valid = False
except ValueError:
header_valid = False
if not header_valid:
err_msg = ('The x-amz-copy-source-range value must be of the '
'form bytes=first-last where first and last are '
'the zero-based offsets of the first and last '
'bytes to copy')
raise InvalidArgument('x-amz-source-range', rng, err_msg)
source_size = int(source_resp.headers['Content-Length'])
if not rng_obj.ranges_for_length(source_size):
err_msg = ('Range specified is not valid for source object '
'of size: %s' % source_size)
raise InvalidArgument('x-amz-source-range', rng, err_msg)
req.headers['Range'] = rng
del req.headers['X-Amz-Copy-Source-Range']
if 'X-Amz-Copy-Source' in req.headers:
# Clear some problematic headers that might be on the source
req.headers.update({
sysmeta_header('object', 'etag'):
'',
'X-Object-Sysmeta-Swift3-Etag':
'', # for legacy data
'X-Object-Sysmeta-Slo-Etag':
'',
'X-Object-Sysmeta-Slo-Size':
'',
'X-Object-Sysmeta-Container-Update-Override-Etag':
'',
})
log_s3api_command(req, method)
resp = req.get_response(self.app)
if 'X-Amz-Copy-Source' in req.headers:
resp.append_copy_resp_body(req.controller_name,
req_timestamp.s3xmlformat)
resp.status = 200
return resp
def POST(self, req):
"""
Handles Complete Multipart Upload.
"""
log_s3api_command(req, 'complete-multipart-upload')
upload_id = req.params['uploadId']
resp = _get_upload_info(req, self.app, upload_id)
headers = {}
for key, val in resp.headers.iteritems():
_key = key.lower()
if _key.startswith('x-amz-meta-'):
headers['x-object-meta-' + _key[11:]] = val
elif _key == 'content-type':
headers['Content-Type'] = val
for key, val in resp.sysmeta_headers.items():
_key = key.lower()
if _key == OBJECT_TAGGING_HEADER.lower():
headers[key] = val
# Query for the objects in the segments area to make sure it completed
query = {
'format': 'json',
'prefix': '%s/%s/' % (req.object_name, upload_id),
'delimiter': '/'
}
# Force the master to be sure to fetch all uploaded parts
req.environ.setdefault('oio.query', {})
req.environ['oio.query']['force_master'] = True
container = req.container_name + MULTIUPLOAD_SUFFIX
resp = req.get_response(self.app, 'GET', container, '', query=query)
objinfo = json.loads(resp.body)
# pylint: disable-msg=no-member
objinfo.sort(key=lambda o: int(o['name'].split('/')[-1]))
objtable = dict((o['name'].encode('utf-8'), {
'path': '/'.join(['', container, o['name']]),
'etag': o['hash'],
'size_bytes': o['bytes']
}) for o in objinfo)
s3_etag_hasher = md5()
manifest = []
previous_number = 0
try:
xml = req.xml(MAX_COMPLETE_UPLOAD_BODY_SIZE)
if not xml:
raise InvalidRequest(msg='You must specify at least one part')
if 'content-md5' in req.headers:
# If an MD5 was provided, we need to verify it.
# Note that S3Request already took care of translating to ETag
if req.headers['etag'] != md5(xml).hexdigest():
raise BadDigest(content_md5=req.headers['content-md5'])
# We're only interested in the body here, in the
# multipart-upload controller -- *don't* let it get
# plumbed down to the object-server
del req.headers['etag']
complete_elem = fromstring(xml, 'CompleteMultipartUpload')
for part_elem in complete_elem.iterchildren('Part'):
part_number = int(part_elem.find('./PartNumber').text)
if part_number <= previous_number:
raise InvalidPartOrder(upload_id=upload_id)
previous_number = part_number
etag = part_elem.find('./ETag').text
if len(etag) >= 2 and etag[0] == '"' and etag[-1] == '"':
# strip double quotes
etag = etag[1:-1]
info = objtable.get("%s/%s/%s" %
(req.object_name, upload_id, part_number))
if info is None or info['etag'] != etag:
raise InvalidPart(upload_id=upload_id,
part_number=part_number)
s3_etag_hasher.update(binascii.a2b_hex(etag))
info['size_bytes'] = int(info['size_bytes'])
manifest.append(info)
except (XMLSyntaxError, DocumentInvalid):
raise MalformedXML()
except ErrorResponse:
raise
except Exception as e:
LOGGER.error(e)
raise
s3_etag = '%s-%d' % (s3_etag_hasher.hexdigest(), len(manifest))
headers[sysmeta_header('object', 'etag')] = s3_etag
# Leave base header value blank; SLO will populate
c_etag = '; s3_etag=%s' % s3_etag
headers['X-Object-Sysmeta-Container-Update-Override-Etag'] = c_etag
# Following swift commit 7f636a5, zero-byte segments aren't allowed,
# even as the final segment
empty_seg = None
if manifest[-1]['size_bytes'] == 0:
empty_seg = manifest.pop()
# We'll check the sizes of all except the last segment below, but
# since we just popped off a zero-byte segment, we should check
# that last segment, too.
if manifest and manifest[-1]['size_bytes'] < CONF.min_segment_size:
raise EntityTooSmall()
# Check the size of each segment except the last and make sure they are
# all more than the minimum upload chunk size
for info in manifest[:-1]:
if info['size_bytes'] < CONF.min_segment_size:
raise EntityTooSmall()
try:
# TODO: add support for versioning
if manifest:
resp = req.get_response(self.app,
'PUT',
body=json.dumps(manifest),
query={'multipart-manifest': 'put'},
headers=headers)
else:
# the upload must have consisted of a single zero-length part
# just write it directly
resp = req.get_response(self.app,
'PUT',
body='',
headers=headers)
except ErrorResponse as e:
msg = str(e._msg)
expected_msg = 'too small; each segment must be at least 1 byte'
if expected_msg in msg:
# FIXME: AWS S3 allows a smaller object than 5 MB if there is
# only one part. Use a COPY request to copy the part object
# from the segments container instead.
raise EntityTooSmall(msg)
else:
raise
if empty_seg:
# clean up the zero-byte segment
_, empty_seg_cont, empty_seg_name = empty_seg['path'].split('/', 2)
req.get_response(self.app,
'DELETE',
container=empty_seg_cont,
obj=empty_seg_name)
# clean up the multipart-upload record
obj = '%s/%s' % (req.object_name, upload_id)
req.environ['oio.ephemeral_object'] = True
req.get_response(self.app, 'DELETE', container, obj)
result_elem = Element('CompleteMultipartUploadResult')
# NOTE: boto with sig v4 appends port to HTTP_HOST value at the
# request header when the port is non default value and it makes
# req.host_url like as http://localhost:8080:8080/path
# that obviously invalid. Probably it should be resolved at
# swift.common.swob though, tentatively we are parsing and
# reconstructing the correct host_url info here.
# in detail, https://github.com/boto/boto/pull/3513
parsed_url = urlparse(req.host_url)
host_url = '%s://%s' % (parsed_url.scheme, parsed_url.hostname)
if parsed_url.port:
host_url += ':%s' % parsed_url.port
SubElement(result_elem, 'Location').text = host_url + req.path
SubElement(result_elem, 'Bucket').text = req.container_name
SubElement(result_elem, 'Key').text = req.object_name
SubElement(result_elem, 'ETag').text = '"%s"' % s3_etag
del resp.headers['ETag']
resp.body = tostring(result_elem)
resp.status = 200
resp.content_type = "application/xml"
return resp
def GET(self, req):
"""
Handles List Parts.
"""
log_s3api_command(req, 'list-parts')
def filter_part_num_marker(o):
try:
num = int(os.path.basename(o['name']))
return num > part_num_marker
except ValueError:
return False
encoding_type = req.params.get('encoding-type')
if encoding_type is not None and encoding_type != 'url':
err_msg = 'Invalid Encoding Method specified in Request'
raise InvalidArgument('encoding-type', encoding_type, err_msg)
upload_id = req.params['uploadId']
_check_upload_info(req, self.app, upload_id)
maxparts = req.get_validated_param('max-parts',
DEFAULT_MAX_PARTS_LISTING,
CONF.max_parts_listing)
part_num_marker = req.get_validated_param('part-number-marker', 0)
query = {
'format': 'json',
'limit': maxparts + 1,
'prefix': '%s/%s/' % (req.object_name, upload_id),
'delimiter': '/'
}
container = req.container_name + MULTIUPLOAD_SUFFIX
resp = req.get_response(self.app,
container=container,
obj='',
query=query)
objects = json.loads(resp.body)
last_part = 0
# If the caller requested a list starting at a specific part number,
# construct a sub-set of the object list.
objList = filter(filter_part_num_marker, objects)
# pylint: disable-msg=E1103
objList.sort(key=lambda o: int(o['name'].split('/')[-1]))
if len(objList) > maxparts:
objList = objList[:maxparts]
truncated = True
else:
truncated = False
# TODO: We have to retrieve object list again when truncated is True
# and some objects filtered by invalid name because there could be no
# enough objects for limit defined by maxparts.
if objList:
o = objList[-1]
last_part = os.path.basename(o['name'])
result_elem = Element('ListPartsResult')
if encoding_type is not None:
result_elem.encoding_type = encoding_type
SubElement(result_elem, 'Bucket').text = req.container_name
SubElement(result_elem, 'Key').text = req.object_name
SubElement(result_elem, 'UploadId').text = upload_id
initiator_elem = SubElement(result_elem, 'Initiator')
SubElement(initiator_elem, 'ID').text = req.user_id
SubElement(initiator_elem, 'DisplayName').text = req.user_id
owner_elem = SubElement(result_elem, 'Owner')
SubElement(owner_elem, 'ID').text = req.user_id
SubElement(owner_elem, 'DisplayName').text = req.user_id
SubElement(result_elem, 'StorageClass').text = 'STANDARD'
SubElement(result_elem, 'PartNumberMarker').text = str(part_num_marker)
SubElement(result_elem, 'NextPartNumberMarker').text = str(last_part)
SubElement(result_elem, 'MaxParts').text = str(maxparts)
if 'encoding-type' in req.params:
SubElement(result_elem, 'EncodingType').text = \
req.params['encoding-type']
SubElement(result_elem, 'IsTruncated').text = \
'true' if truncated else 'false'
for i in objList:
part_elem = SubElement(result_elem, 'Part')
SubElement(part_elem, 'PartNumber').text = i['name'].split('/')[-1]
SubElement(part_elem, 'LastModified').text = \
i['last_modified'][:-3] + 'Z'
SubElement(part_elem, 'ETag').text = '"%s"' % i['hash']
SubElement(part_elem, 'Size').text = str(i['bytes'])
body = tostring(result_elem, encoding_type=encoding_type)
return HTTPOk(body=body, content_type='application/xml')
def GET(self, req):
"""
Handles List Multipart Uploads
"""
log_s3api_command(req, 'list-multipart-uploads')
def separate_uploads(uploads, prefix, delimiter):
"""
separate_uploads will separate uploads into non_delimited_uploads
(a subset of uploads) and common_prefixes according to the
specified delimiter. non_delimited_uploads is a list of uploads
which exclude the delimiter. common_prefixes is a set of prefixes
prior to the specified delimiter. Note that the prefix in the
common_prefixes includes the delimiter itself.
i.e. if '/' delimiter specified and then the uploads is consists of
['foo', 'foo/bar'], this function will return (['foo'], ['foo/']).
:param uploads: A list of uploads dictionary
:param prefix: A string of prefix reserved on the upload path.
(i.e. the delimiter must be searched behind the
prefix)
:param delimiter: A string of delimiter to split the path in each
upload
:return (non_delimited_uploads, common_prefixes)
"""
(prefix, delimiter) = \
utf8encode(prefix, delimiter)
non_delimited_uploads = []
common_prefixes = set()
for upload in uploads:
key = upload['key']
end = key.find(delimiter, len(prefix))
if end >= 0:
common_prefix = key[:end + len(delimiter)]
common_prefixes.add(common_prefix)
else:
non_delimited_uploads.append(upload)
return non_delimited_uploads, sorted(common_prefixes)
encoding_type = req.params.get('encoding-type')
if encoding_type is not None and encoding_type != 'url':
err_msg = 'Invalid Encoding Method specified in Request'
raise InvalidArgument('encoding-type', encoding_type, err_msg)
keymarker = req.params.get('key-marker', '')
uploadid = req.params.get('upload-id-marker', '')
maxuploads = req.get_validated_param('max-uploads',
DEFAULT_MAX_UPLOADS,
DEFAULT_MAX_UPLOADS)
query = {
'format': 'json',
'limit': maxuploads + 1,
}
if uploadid and keymarker:
query.update({'marker': '%s/%s' % (keymarker, uploadid)})
elif keymarker:
query.update({'marker': '%s/~' % (keymarker)})
if 'prefix' in req.params:
query.update({'prefix': req.params['prefix']})
container = req.container_name + MULTIUPLOAD_SUFFIX
try:
req.environ['oio.list_mpu'] = True
resp = req.get_response(self.app, container=container, query=query)
objects = json.loads(resp.body)
except NoSuchBucket:
# Assume NoSuchBucket as no uploads
objects = []
def object_to_upload(object_info):
obj, upid = object_info['name'].rsplit('/', 1)
obj_dict = {
'key': obj,
'upload_id': upid,
'last_modified': object_info['last_modified']
}
return obj_dict
# uploads is a list consists of dict, {key, upload_id, last_modified}
# Note that pattern matcher will drop whole segments objects like as
# object_name/upload_id/1.
pattern = re.compile('/[0-9]+$')
uploads = [
object_to_upload(obj) for obj in objects
if pattern.search(obj.get('name', '')) is None
]
prefixes = []
if 'delimiter' in req.params:
prefix = req.params.get('prefix', '')
delimiter = req.params['delimiter']
uploads, prefixes = \
separate_uploads(uploads, prefix, delimiter)
if len(uploads) > maxuploads:
uploads = uploads[:maxuploads]
truncated = True
else:
truncated = False
nextkeymarker = ''
nextuploadmarker = ''
if len(uploads) > 1:
nextuploadmarker = uploads[-1]['upload_id']
nextkeymarker = uploads[-1]['key']
result_elem = Element('ListMultipartUploadsResult')
if encoding_type is not None:
result_elem.encoding_type = encoding_type
SubElement(result_elem, 'Bucket').text = req.container_name
SubElement(result_elem, 'KeyMarker').text = keymarker
SubElement(result_elem, 'UploadIdMarker').text = uploadid
SubElement(result_elem, 'NextKeyMarker').text = nextkeymarker
SubElement(result_elem, 'NextUploadIdMarker').text = nextuploadmarker
if 'delimiter' in req.params:
SubElement(result_elem, 'Delimiter').text = \
req.params['delimiter']
if 'prefix' in req.params:
SubElement(result_elem, 'Prefix').text = req.params['prefix']
SubElement(result_elem, 'MaxUploads').text = str(maxuploads)
if encoding_type is not None:
SubElement(result_elem, 'EncodingType').text = encoding_type
SubElement(result_elem, 'IsTruncated').text = \
'true' if truncated else 'false'
# TODO: don't show uploads which are initiated before this bucket is
# created.
for u in uploads:
upload_elem = SubElement(result_elem, 'Upload')
SubElement(upload_elem, 'Key').text = u['key']
SubElement(upload_elem, 'UploadId').text = u['upload_id']
initiator_elem = SubElement(upload_elem, 'Initiator')
SubElement(initiator_elem, 'ID').text = req.user_id
SubElement(initiator_elem, 'DisplayName').text = req.user_id
owner_elem = SubElement(upload_elem, 'Owner')
SubElement(owner_elem, 'ID').text = req.user_id
SubElement(owner_elem, 'DisplayName').text = req.user_id
SubElement(upload_elem, 'StorageClass').text = 'STANDARD'
SubElement(upload_elem, 'Initiated').text = \
u['last_modified'][:-3] + 'Z'
for p in prefixes:
elem = SubElement(result_elem, 'CommonPrefixes')
SubElement(elem, 'Prefix').text = p
body = tostring(result_elem, encoding_type=encoding_type)
return HTTPOk(body=body, content_type='application/xml')
def HEAD(self, req):
"""
Handles Head Part (regular HEAD but with ?part-number=N).
"""
log_s3api_command(req, 'head-object-part')
return self.GETorHEAD(req)
def GET(self, req):
"""
Handles Get Part (regular Get but with ?part-number=N).
"""
log_s3api_command(req, 'get-object-part')
return self.GETorHEAD(req)
def GET(self, req):
"""
Handle GET Bucket (List Objects) request
"""
max_keys = req.get_validated_param('max-keys', CONF.max_bucket_listing)
# TODO: Separate max_bucket_listing and default_bucket_listing
tag_max_keys = max_keys
max_keys = min(max_keys, CONF.max_bucket_listing)
encoding_type = req.params.get('encoding-type')
if encoding_type is not None and encoding_type != 'url':
err_msg = 'Invalid Encoding Method specified in Request'
raise InvalidArgument('encoding-type', encoding_type, err_msg)
query = {
'format': 'json',
'limit': max_keys + 1,
}
if 'marker' in req.params:
query.update({'marker': req.params['marker']})
if 'prefix' in req.params:
query.update({'prefix': req.params['prefix']})
if 'delimiter' in req.params:
query.update({'delimiter': req.params['delimiter']})
# GET Bucket (List Objects) Version 2 parameters
is_v2 = int(req.params.get('list-type', '1')) == 2
fetch_owner = False
if is_v2:
log_s3api_command(req, 'list-objects-v2')
if 'start-after' in req.params:
query.update({'marker': req.params['start-after']})
# continuation-token overrides start-after
if 'continuation-token' in req.params:
decoded = b64decode(req.params['continuation-token'])
query.update({'marker': decoded})
if 'fetch-owner' in req.params:
fetch_owner = config_true_value(req.params['fetch-owner'])
else:
log_s3api_command(req, 'list-objects')
resp = req.get_response(self.app, query=query)
objects = json.loads(resp.body)
if 'versions' in req.params:
req.container_name += VERSIONING_SUFFIX
query['reverse'] = 'true'
try:
resp = req.get_response(self.app, query=query)
versioned_objects = json.loads(resp.body)
prefixes = set()
for o in versioned_objects:
if 'name' in o:
# The name looks like this:
# '%03x%s/%s' % (len(name), name, version)
o['name'], o['version_id'] = \
o['name'][3:].rsplit('/', 1)
else:
prefixes.add(o['subdir'])
# suppress duplicated prefixes
for o in list(objects):
if 'subdir' in o and o['subdir'] in prefixes:
objects.remove(o)
objects.extend(versioned_objects)
except NoSuchBucket:
# the bucket may not be versioned
pass
req.container_name = req.container_name[:-len(VERSIONING_SUFFIX)]
objects.sort(key=lambda o: o.get('name') or o.get('subdir'))
for o in objects:
if 'subdir' not in o and not o.get('version_id'):
info = req.get_object_info(
self.app, object_name=o['name'].encode('utf-8'))
o['sysmeta_version_id'] = info.get('sysmeta', {}).get(
'version-id', 'null')
if 'versions' in req.params:
elem = Element('ListVersionsResult')
else:
elem = Element('ListBucketResult')
if encoding_type is not None:
elem.encoding_type = encoding_type
SubElement(elem, 'Name').text = req.container_name
SubElement(elem, 'Prefix').text = req.params.get('prefix')
# Filter objects according to version-id-marker and key-marker
v_marker = req.params.get('version-id-marker')
k_marker = req.params.get('key-marker')
k_marker_matched = not bool(k_marker)
if 'versions' in req.params and (v_marker or k_marker):
to_delete = []
for i, o in enumerate(objects):
if 'subdir' not in o:
version_id = o.get('version_id',
o.get('sysmeta_version_id', 'null'))
if not k_marker_matched and k_marker != o['name']:
to_delete.append(i)
if k_marker == o['name']:
k_marker_matched = True
if k_marker == o['name'] and v_marker:
if v_marker == version_id:
v_marker = None
to_delete.append(i)
for i in reversed(to_delete):
objects.pop(i)
# in order to judge that truncated is valid, check whether
# max_keys + 1 th element exists in swift.
is_truncated = max_keys > 0 and len(objects) > max_keys
objects = objects[:max_keys]
if not is_v2:
if 'versions' in req.params:
SubElement(elem, 'KeyMarker').text = req.params.get(
'key-marker')
SubElement(elem, 'VersionIdMarker').text = req.params.get(
'version-id-marker')
else:
SubElement(elem, 'Marker').text = req.params.get('marker')
if is_truncated and 'delimiter' in req.params:
if 'name' in objects[-1]:
SubElement(elem, 'NextMarker').text = \
objects[-1]['name']
if 'subdir' in objects[-1]:
SubElement(elem, 'NextMarker').text = \
objects[-1]['subdir']
else:
if is_truncated:
if 'name' in objects[-1]:
SubElement(elem, 'NextContinuationToken').text = \
b64encode(objects[-1]['name'].encode('utf8'))
if 'subdir' in objects[-1]:
SubElement(elem, 'NextContinuationToken').text = \
b64encode(objects[-1]['subdir'].encode('utf8'))
if 'continuation-token' in req.params:
SubElement(elem, 'ContinuationToken').text = \
req.params['continuation-token']
if 'start-after' in req.params:
SubElement(elem, 'StartAfter').text = \
req.params['start-after']
SubElement(elem, 'KeyCount').text = str(len(objects))
SubElement(elem, 'MaxKeys').text = str(tag_max_keys)
if 'delimiter' in req.params:
SubElement(elem, 'Delimiter').text = req.params['delimiter']
if encoding_type is not None:
SubElement(elem, 'EncodingType').text = encoding_type
SubElement(elem, 'IsTruncated').text = \
'true' if is_truncated else 'false'
for o in objects:
if 'subdir' not in o:
if 'versions' in req.params:
version_id = o.get('version_id',
o.get('sysmeta_version_id', 'null'))
if o.get('content_type') == DELETE_MARKER_CONTENT_TYPE:
contents = SubElement(elem, 'DeleteMarker')
else:
contents = SubElement(elem, 'Version')
SubElement(contents, 'Key').text = \
o['name'].encode('utf-8')
SubElement(contents, 'VersionId').text = version_id
SubElement(contents, 'IsLatest').text = str(
'version_id' not in o).lower()
else:
contents = SubElement(elem, 'Contents')
SubElement(contents, 'Key').text = \
o['name'].encode('utf-8')
SubElement(contents, 'LastModified').text = \
o['last_modified'][:-3] + 'Z'
if contents.tag != 'DeleteMarker':
if 's3_etag' in o:
# New-enough MUs are already in the right format
etag = o['s3_etag']
elif 'slo_etag' in o:
# SLOs may be in something *close* to the MU format
etag = '"%s-N"' % o['slo_etag'].strip('"')
else:
etag = o['hash']
if len(etag) < 2 or etag[::len(etag) - 1] != '""':
# Normal objects just use the MD5
etag = '"%s"' % o['hash']
# This also catches sufficiently-old SLOs,
# but we have no way to identify those
# from container listings
# Otherwise, somebody somewhere (proxyfs, maybe?)
# made this look like an RFC-compliant ETag;
# we don't need to quote-wrap.
SubElement(contents, 'ETag').text = etag
SubElement(contents, 'Size').text = str(o['bytes'])
if fetch_owner or not is_v2:
owner = SubElement(contents, 'Owner')
SubElement(owner, 'ID').text = req.user_id
SubElement(owner, 'DisplayName').text = req.user_id
if contents.tag != 'DeleteMarker':
SubElement(contents, 'StorageClass').text = 'STANDARD'
for o in objects:
if 'subdir' in o:
common_prefixes = SubElement(elem, 'CommonPrefixes')
SubElement(common_prefixes, 'Prefix').text = \
o['subdir'].encode('utf-8')
body = tostring(elem, encoding_type=encoding_type)
resp = HTTPOk(body=body, content_type='application/xml')
origin = req.headers.get('Origin')
if origin:
rule = get_cors(self.app, req, "GET", origin)
if rule:
cors_fill_headers(req, resp, rule)
return resp
def POST(self, req):
"""
Handles Delete Multiple Objects.
"""
log_s3api_command(req, 'delete-objects')
def object_key_iter(elem):
for obj in elem.iterchildren('Object'):
key = obj.find('./Key').text
if not key:
raise UserKeyMustBeSpecified()
version = obj.find('./VersionId')
if version is not None:
version = version.text
yield key, version
try:
xml = req.xml(MAX_MULTI_DELETE_BODY_SIZE)
if not xml:
raise MissingRequestBodyError()
req.check_md5(xml)
elem = fromstring(xml, 'Delete')
quiet = elem.find('./Quiet')
self.quiet = quiet is not None and quiet.text.lower() == 'true'
delete_list = list(object_key_iter(elem))
if len(delete_list) > CONF.max_multi_delete_objects:
raise MalformedXML()
except (XMLSyntaxError, DocumentInvalid):
raise MalformedXML()
except ErrorResponse:
raise
except Exception as e:
LOGGER.error(e)
raise
elem = Element('DeleteResult')
# check bucket existence
try:
req.get_response(self.app, 'HEAD')
except AccessDenied as error:
body = self._gen_error_body(error, elem, delete_list)
return HTTPOk(body=body)
for key, version in delete_list:
if version is not None:
# TODO: delete the specific version of the object
raise S3NotImplemented()
req.object_name = key
try:
query = req.gen_multipart_manifest_delete_query(self.app)
req.get_response(self.app, method='DELETE', query=query)
except ErrorResponse as e:
error = SubElement(elem, 'Error')
SubElement(error, 'Key').text = key
SubElement(error, 'Code').text = e.__class__.__name__
SubElement(error, 'Message').text = e._msg
continue
if not self.quiet:
deleted = SubElement(elem, 'Deleted')
SubElement(deleted, 'Key').text = key
body = tostring(elem)
return HTTPOk(body=body)
def GET(self, req):
"""
Handle GET Object request
"""
log_s3api_command(req, 'get-object')
return self.GETorHEAD(req)
