def _gen_error_body(self, error, elem, delete_list):
for key, version in delete_list:
if version is not None:
# TODO: delete the specific version of the object
raise S3NotImplemented()
error_elem = SubElement(elem, 'Error')
SubElement(error_elem, 'Key').text = key
SubElement(error_elem, 'Code').text = error.__class__.__name__
SubElement(error_elem, 'Message').text = error._msg
return tostring(elem)
def from_elem(elem):
type = elem.get('{%s}type' % XMLNS_XSI)
if type == 'CanonicalUser':
value = elem.find('./ID').text
return User(value)
elif type == 'Group':
value = elem.find('./URI').text
subclass = get_group_subclass_from_uri(value)
return subclass()
elif type == 'AmazonCustomerByEmail':
raise S3NotImplemented()
else:
raise MalformedACLError()
def version_id_param(req):
"""
Get the version ID specified by the request, if any.
"""
version_id = req.params.get('versionId')
if version_id not in ('null', None):
obj_vers_info = get_swift_info().get('object_versioning')
if obj_vers_info is None:
raise S3NotImplemented()
is_valid_version = obj_vers_info.get('is_valid_version_id',
lambda x: True)
if not is_valid_version(version_id):
raise InvalidArgument('versionId', version_id,
'Invalid version id specified')
return version_id
def swift_acl_translate(acl, group='', user='', xml=False):
"""
Takes an S3 style ACL and returns a list of header/value pairs that
implement that ACL in Swift, or "NotImplemented" if there isn't a way to do
that yet.
"""
swift_acl = {}
swift_acl['public-read'] = [['X-Container-Read', '.r:*,.rlistings']]
# Swift does not support public write:
# https://answers.launchpad.net/swift/+question/169541
swift_acl['public-read-write'] = [['X-Container-Write', '.r:*'],
['X-Container-Read', '.r:*,.rlistings']]
# TODO: if there's a way to get group and user, this should work for
# private:
# swift_acl['private'] = \
# [['HTTP_X_CONTAINER_WRITE', group + ':' + user], \
# ['HTTP_X_CONTAINER_READ', group + ':' + user]]
swift_acl['private'] = [['X-Container-Write', '.'],
['X-Container-Read', '.']]
if xml:
# We are working with XML and need to parse it
try:
elem = fromstring(acl, 'AccessControlPolicy')
except (XMLSyntaxError, DocumentInvalid):
raise MalformedACLError()
acl = 'unknown'
for grant in elem.findall('./AccessControlList/Grant'):
permission = grant.find('./Permission').text
grantee = grant.find('./Grantee').get('{%s}type' % XMLNS_XSI)
if permission == "FULL_CONTROL" and grantee == 'CanonicalUser' and\
acl != 'public-read' and acl != 'public-read-write':
acl = 'private'
elif permission == "READ" and grantee == 'Group' and\
acl != 'public-read-write':
acl = 'public-read'
elif permission == "WRITE" and grantee == 'Group':
acl = 'public-read-write'
else:
acl = 'unsupported'
if acl == 'authenticated-read':
raise S3NotImplemented()
elif acl not in swift_acl:
raise ACLError()
return swift_acl[acl]
def DELETE(self, req):
"""
Handle DELETE Object request
"""
if 'versionId' in req.params and \
req.params['versionId'] != 'null' and \
'object_versioning' not in get_swift_info():
raise S3NotImplemented()
version_id = req.params.get('versionId')
if version_id not in ('null', None):
container_info = req.get_container_info(self.app)
if not container_info.get('sysmeta', {}).get(
'versions-container', ''):
# Versioning has never been enabled
return HTTPNoContent(headers={'x-amz-version-id': version_id})
try:
try:
query = req.gen_multipart_manifest_delete_query(
self.app, version=version_id)
except NoSuchKey:
query = {}
req.headers['Content-Type'] = None # Ignore client content-type
if version_id is not None:
query['version-id'] = version_id
query['symlink'] = 'get'
resp = req.get_response(self.app, query=query)
if query.get('multipart-manifest') and resp.status_int == HTTP_OK:
for chunk in resp.app_iter:
pass # drain the bulk-deleter response
resp.status = HTTP_NO_CONTENT
resp.body = b''
if resp.sw_headers.get('X-Object-Current-Version-Id') == 'null':
new_resp = self._restore_on_delete(req)
if new_resp:
resp = new_resp
except NoSuchKey:
# expect to raise NoSuchBucket when the bucket doesn't exist
req.get_container_info(self.app)
# else -- it's gone! Success.
return HTTPNoContent()
return resp
def from_header(grantee):
"""
Convert a grantee string in the HTTP header to an Grantee instance.
"""
type, value = grantee.split('=', 1)
value = value.strip('"\'')
if type == 'id':
return User(value)
elif type == 'emailAddress':
raise S3NotImplemented()
elif type == 'uri':
# return a subclass instance of Group class
subclass = get_group_subclass_from_uri(value)
return subclass()
else:
raise InvalidArgument(type, value,
'Argument format not recognized')
def GETorHEAD(self, req):
had_match = False
for match_header in ('if-match', 'if-none-match'):
if match_header not in req.headers:
continue
had_match = True
for value in list_from_csv(req.headers[match_header]):
value = normalize_etag(value)
if value.endswith('-N'):
# Deal with fake S3-like etags for SLOs uploaded via Swift
req.headers[match_header] += ', ' + value[:-2]
if had_match:
# Update where to look
update_etag_is_at_header(req, sysmeta_header('object', 'etag'))
object_name = req.object_name
version_id = req.params.get('versionId')
if version_id not in ('null', None) and \
'object_versioning' not in get_swift_info():
raise S3NotImplemented()
query = {} if version_id is None else {'version-id': version_id}
if version_id not in ('null', None):
container_info = req.get_container_info(self.app)
if not container_info.get('sysmeta', {}).get(
'versions-container', ''):
# Versioning has never been enabled
raise NoSuchVersion(object_name, version_id)
resp = req.get_response(self.app, query=query)
if req.method == 'HEAD':
resp.app_iter = None
if 'x-amz-meta-deleted' in resp.headers:
raise NoSuchKey(object_name)
for key in ('content-type', 'content-language', 'expires',
'cache-control', 'content-disposition',
'content-encoding'):
if 'response-' + key in req.params:
resp.headers[key] = req.params['response-' + key]
return resp
def PUT(self, req):
"""
Handles PUT Bucket acl and PUT Object acl.
"""
if req.is_object_request:
self.set_s3api_command(req, 'put-object-acl')
else:
self.set_s3api_command(req, 'put-bucket-acl')
if req.is_object_request:
# Handle Object ACL
raise S3NotImplemented()
else:
# Handle Bucket ACL
xml = req.xml(MAX_ACL_BODY_SIZE)
if all(['HTTP_X_AMZ_ACL' in req.environ, xml]):
# S3 doesn't allow to give ACL with both ACL header and body.
raise UnexpectedContent()
elif not any(['HTTP_X_AMZ_ACL' in req.environ, xml]):
# Both canned ACL header and xml body are missing
raise MissingSecurityHeader(missing_header_name='x-amz-acl')
else:
# correct ACL exists in the request
if xml:
# We very likely have an XML-based ACL request.
# let's try to translate to the request header
try:
translated_acl = swift_acl_translate(xml, xml=True)
except ACLError:
raise MalformedACLError()
for header, acl in translated_acl:
req.headers[header] = acl
resp = req.get_response(self.app, 'POST')
resp.status = HTTP_OK
resp.headers.update({'Location': req.container_name})
return resp
def POST(self, req):
raise S3NotImplemented()
def elem(self):
"""
Get an etree element of this instance.
"""
raise S3NotImplemented()
def __init__(self, app, conf, logger, **kwargs):
raise S3NotImplemented('The requested resource is not implemented')
def POST(self, req):
"""
Handle POST Bucket request
"""
raise S3NotImplemented()
def POST(self, req):
"""
Handles Delete Multiple Objects.
"""
def object_key_iter(elem):
for obj in elem.iterchildren('Object'):
key = obj.find('./Key').text
if not key:
raise UserKeyMustBeSpecified()
version = obj.find('./VersionId')
if version is not None:
version = version.text
yield key, version
max_body_size = min(
# FWIW, AWS limits multideletes to 1000 keys, and swift limits
# object names to 1024 bytes (by default). Add a factor of two to
# allow some slop.
2 * self.conf.max_multi_delete_objects * MAX_OBJECT_NAME_LENGTH,
# But, don't let operators shoot themselves in the foot
10 * 1024 * 1024)
try:
xml = req.xml(max_body_size)
if not xml:
raise MissingRequestBodyError()
req.check_md5(xml)
elem = fromstring(xml, 'Delete', self.logger)
quiet = elem.find('./Quiet')
if quiet is not None and quiet.text.lower() == 'true':
self.quiet = True
else:
self.quiet = False
delete_list = list(object_key_iter(elem))
if len(delete_list) > self.conf.max_multi_delete_objects:
raise MalformedXML()
except (XMLSyntaxError, DocumentInvalid):
raise MalformedXML()
except ErrorResponse:
raise
except Exception as e:
self.logger.error(e)
raise
elem = Element('DeleteResult')
# check bucket existence
try:
req.get_response(self.app, 'HEAD')
except AccessDenied as error:
body = self._gen_error_body(error, elem, delete_list)
return HTTPOk(body=body)
for key, version in delete_list:
if version is not None:
# TODO: delete the specific version of the object
raise S3NotImplemented()
req.object_name = key
try:
query = req.gen_multipart_manifest_delete_query(self.app)
req.get_response(self.app, method='DELETE', query=query)
except NoSuchKey:
pass
except ErrorResponse as e:
error = SubElement(elem, 'Error')
SubElement(error, 'Key').text = key
SubElement(error, 'Code').text = e.__class__.__name__
SubElement(error, 'Message').text = e._msg
continue
if not self.quiet:
deleted = SubElement(elem, 'Deleted')
SubElement(deleted, 'Key').text = key
body = tostring(elem)
return HTTPOk(body=body)
def POST(self, req):
"""
Handles Delete Multiple Objects.
"""
def object_key_iter(elem):
for obj in elem.iterchildren('Object'):
key = obj.find('./Key').text
if not key:
raise UserKeyMustBeSpecified()
version = obj.find('./VersionId')
if version is not None:
version = version.text
yield key, version
try:
xml = req.xml(MAX_MULTI_DELETE_BODY_SIZE)
if not xml:
raise MissingRequestBodyError()
req.check_md5(xml)
elem = fromstring(xml, 'Delete', self.logger)
quiet = elem.find('./Quiet')
if quiet is not None and quiet.text.lower() == 'true':
self.quiet = True
else:
self.quiet = False
delete_list = list(object_key_iter(elem))
if len(delete_list) > self.conf.max_multi_delete_objects:
raise MalformedXML()
except (XMLSyntaxError, DocumentInvalid):
raise MalformedXML()
except ErrorResponse:
raise
except Exception as e:
exc_type, exc_value, exc_traceback = sys.exc_info()
self.logger.error(e)
raise exc_type, exc_value, exc_traceback
elem = Element('DeleteResult')
# check bucket existence
try:
req.get_response(self.app, 'HEAD')
except AccessDenied as error:
body = self._gen_error_body(error, elem, delete_list)
return HTTPOk(body=body)
for key, version in delete_list:
if version is not None:
# TODO: delete the specific version of the object
raise S3NotImplemented()
req.object_name = key
try:
query = req.gen_multipart_manifest_delete_query(self.app)
req.get_response(self.app, method='DELETE', query=query)
except NoSuchKey:
pass
except ErrorResponse as e:
error = SubElement(elem, 'Error')
SubElement(error, 'Key').text = key
SubElement(error, 'Code').text = e.__class__.__name__
SubElement(error, 'Message').text = e._msg
continue
if not self.quiet:
deleted = SubElement(elem, 'Deleted')
SubElement(deleted, 'Key').text = key
body = tostring(elem)
return HTTPOk(body=body)
def POST(self, req):
"""
Handles Delete Multiple Objects.
"""
def object_key_iter(elem):
for obj in elem.iterchildren('Object'):
key = obj.find('./Key').text
if not key:
raise UserKeyMustBeSpecified()
version = obj.find('./VersionId')
if version is not None:
version = version.text
yield key, version
max_body_size = min(
# FWIW, AWS limits multideletes to 1000 keys, and swift limits
# object names to 1024 bytes (by default). Add a factor of two to
# allow some slop.
2 * self.conf.max_multi_delete_objects * MAX_OBJECT_NAME_LENGTH,
# But, don't let operators shoot themselves in the foot
10 * 1024 * 1024)
try:
xml = req.xml(max_body_size)
if not xml:
raise MissingRequestBodyError()
req.check_md5(xml)
elem = fromstring(xml, 'Delete', self.logger)
quiet = elem.find('./Quiet')
if quiet is not None and quiet.text.lower() == 'true':
self.quiet = True
else:
self.quiet = False
delete_list = list(object_key_iter(elem))
if len(delete_list) > self.conf.max_multi_delete_objects:
raise MalformedXML()
except (XMLSyntaxError, DocumentInvalid):
raise MalformedXML()
except ErrorResponse:
raise
except Exception as e:
self.logger.error(e)
raise
elem = Element('DeleteResult')
# check bucket existence
try:
req.get_response(self.app, 'HEAD')
except AccessDenied as error:
body = self._gen_error_body(error, elem, delete_list)
return HTTPOk(body=body)
if any(version is not None for _key, version in delete_list):
# TODO: support deleting specific versions of objects
raise S3NotImplemented()
def do_delete(base_req, key, version):
req = copy.copy(base_req)
req.environ = copy.copy(base_req.environ)
req.object_name = key
try:
query = req.gen_multipart_manifest_delete_query(self.app)
resp = req.get_response(self.app,
method='DELETE',
query=query,
headers={'Accept': 'application/json'})
# Have to read the response to actually do the SLO delete
if query:
try:
delete_result = json.loads(resp.body)
if delete_result['Errors']:
# NB: bulk includes 404s in "Number Not Found",
# not "Errors"
msg_parts = [delete_result['Response Status']]
msg_parts.extend(
'%s: %s' % (obj, status)
for obj, status in delete_result['Errors'])
return key, {
'code': 'SLODeleteError',
'message': '\n'.join(msg_parts)
}
# else, all good
except (ValueError, TypeError, KeyError):
# Logs get all the gory details
self.logger.exception(
'Could not parse SLO delete response: %r',
resp.body)
# Client gets something more generic
return key, {
'code': 'SLODeleteError',
'message': 'Unexpected swift response'
}
except NoSuchKey:
pass
except ErrorResponse as e:
return key, {'code': e.__class__.__name__, 'message': e._msg}
return key, None
with StreamingPile(self.conf.multi_delete_concurrency) as pile:
for key, err in pile.asyncstarmap(
do_delete,
((req, key, version) for key, version in delete_list)):
if err:
error = SubElement(elem, 'Error')
SubElement(error, 'Key').text = key
SubElement(error, 'Code').text = err['code']
SubElement(error, 'Message').text = err['message']
elif not self.quiet:
deleted = SubElement(elem, 'Deleted')
SubElement(deleted, 'Key').text = key
body = tostring(elem)
return HTTPOk(body=body)
def PUT(self, req):
"""
Handles PUT Bucket versioning.
"""
raise S3NotImplemented()
def __contains__(self, key):
"""
The key argument is a S3 user id. This method checks that the user id
belongs to this class.
"""
raise S3NotImplemented()
