def PUT(self, req):
"""
Handle PUT Bucket website request
"""
xml = req.xml(MAX_PUT_BUCKET_WEBSITE_SIZE)
if xml:
try:
elem = fromstring(xml, 'WebsiteConfiguration')
index = elem.find('IndexDocument')
sufix = index.find('Suffix').text
if elem.find('ErrorDocument') is not None:
error_doc = elem.find('ErrorDocument')
key = error_doc.find('Key').text
# resp = req.get_response(self.app, obj=sufix,method='GET')
# if resp.status_int==HTTP_NOT_FOUND:
# raise NoSuchKey(sufix)
req.headers['X-Container-Meta-Web-Index'] = str(sufix)
req.headers['X-Container-Meta-Web-Error'] = str(key)
req.headers['X-Container-Meta-Web-Listings'] = 'true'
req.headers['X-Container-Meta-Web-Listings-CSS'] = '*.css'
req.headers['X-Container-Read'] = '.r:*'
except (XMLSyntaxError, DocumentInvalid):
raise MalformedXML()
except Exception as e:
exc_type, exc_value, exc_traceback = sys.exc_info()
LOGGER.error(e)
raise exc_type, exc_value, exc_traceback
resp = req.get_response(self.app)
resp.status = HTTP_OK
return resp
def PUT(self, req):
"""
Handle PUT Bucket Referer request
"""
xml = req.xml(MAX_PUT_BUCKET_REFERER_SIZE)
if xml:
# check referer
try:
elem = fromstring(xml, 'RefererConfiguration')
allow_empyt_referer=elem.find('AllowEmptyReferer').text
if allow_empyt_referer not in ['true','false']:
raise InvalidArgument()
referer_list=elem.find('RefererList')
swift_referers=[]
for referer in referer_list.findall('Referer'):
swift_referers.append(referer.text)
if len(swift_referers)==0 :
req.headers['X-Container-Read']=' '
else:
req.headers['X-Container-Read'] = '.r:'+','.join(get_real_url(swift_referers))
except (XMLSyntaxError, DocumentInvalid):
raise MalformedXML()
except Exception as e:
exc_type, exc_value, exc_traceback = sys.exc_info()
LOGGER.error(e)
raise exc_type, exc_value, exc_traceback
resp = req.get_response(self.app)
resp.status = HTTP_OK
return resp
def PUT(self, app):
if self.req.is_object_request:
b_resp = self.req.get_acl_response(app, 'HEAD', obj='')
o_resp = self._handle_acl(app, 'HEAD', permission='WRITE-ACL')
req_acl = get_acl(self.req.headers,
self.req.xml(ACL.max_xml_length),
b_resp.bucket_acl.owner,
o_resp.object_acl.owner)
# Don't change the owner of the resource by PUT acl request.
o_resp.object_acl.check_owner(req_acl.owner.id)
g = req_acl.grant
LOGGER.debug('Grant %s permission on the object /%s/%s' %
(g, self.req.container_name,
self.req.object_name))
if 'X-Oss-Acl' not in self.req.headers:
if req_acl=='private':
req_acl=='default'
try:
resp =self.req.get_acl_response(app, 'GET')
if resp.object_acl:
req_acl=resp.object_acl
except :
pass
self.req.object_acl = req_acl
else:
self._handle_acl(app, self.method)
def PUT(self, req):
"""
Handle PUT Bucket request
"""
location = ""
xml = req.xml(MAX_PUT_BUCKET_BODY_SIZE)
if xml:
# check location
try:
elem = fromstring(xml, 'CreateBucketConfiguration')
location = elem.find('./LocationConstraint').text
except (XMLSyntaxError, DocumentInvalid):
raise MalformedXML()
except Exception as e:
exc_type, exc_value, exc_traceback = sys.exc_info()
LOGGER.error(e)
raise exc_type, exc_value, exc_traceback
if location not in CONF.location:
# Oss2swift cannot support multiple regions currently.
raise InvalidLocationConstraint()
if location is None or location == "":
location = choice(CONF.location)
req.headers['X-Container-Meta-Location'] = location
# req.headers['X-Container-Meta-Location'] = location
req.headers['X-Container-Meta-Create'] = time.time()
resp = req.get_response(self.app)
resp.status = HTTP_OK
resp.location = '/' + location
return resp
def wrapped(self, req):
if not req.is_bucket_request:
if err_resp:
raise err_resp(msg=err_msg)
LOGGER.debug('A key is specified for bucket API.')
req.object_name = None
return func(self, req)
def handle_request(self, req):
LOGGER.debug('Calling Oss2Swift Middleware')
LOGGER.debug(req.__dict__)
controller = req.controller(self.app)
if hasattr(controller, req.method):
handler = getattr(controller, req.method)
if not getattr(handler, 'publicly_accessible', False):
raise MethodNotAllowed(req.method,
req.controller.resource_type())
res = handler(req)
else:
raise MethodNotAllowed(req.method, req.controller.resource_type())
return res
def POST(self, app):
if self.req.is_bucket_request:
resp = self._handle_acl(app, 'HEAD', permission='WRITE-ACL')
req_acl = get_acl(self.req.headers,
self.req.xml(ACL.max_xml_length),
resp.bucket_acl.owner)
# Don't change the owner of the resource by PUT acl request.
resp.bucket_acl.check_owner(req_acl.owner.id)
g = req_acl.grant
LOGGER.debug('Grant %s permission on the bucket /%s' %
(g,self.req.container_name))
self.req.bucket_acl = req_acl
else:
self._handle_acl(app, self.method)
def controller(self):
if self.is_service_request:
return ServiceController
if not self.slo_enabled:
multi_part = ['partNumber', 'uploadId', 'uploads']
if len([p for p in multi_part if p in self.params]):
LOGGER.warning('multipart: No SLO middleware in pipeline')
raise OssNotImplemented("Multi-part feature isn't support")
# if 'objectMeta' in self.params:
# return ObjectController
if 'acl' in self.params:
return AclController
if 'cors' in self.params:
return CorsController
if 'delete' in self.params:
return MultiObjectDeleteController
if 'location' in self.params:
return LocationController
if 'logging' in self.params:
return LoggingStatusController
if 'partNumber' in self.params:
return PartController
if 'uploadId' in self.params:
return UploadController
if 'uploads' in self.params:
return UploadsController
if 'versioning' in self.params:
return VersioningController
if 'lifecycle' in self.params:
return LifecycleController
if 'website' in self.params:
return WebsiteController
if 'referer' in self.params:
return RefererController
unsupported = ('notification', 'policy', 'requestPayment', 'torrent',
'tagging', 'restore')
if set(unsupported) & set(self.params):
return UnsupportedController
if self.is_object_request:
return ObjectController
return BucketController
def __call__(self, env, start_response):
try:
req_class = get_request_class(env)
req = req_class(env, self.app, self.slo_enabled)
resp = self.handle_request(req)
except NotOssRequest:
resp = self.app
except ErrorResponse as err_resp:
if isinstance(err_resp, InternalError):
LOGGER.exception(err_resp)
resp = err_resp
except Exception as e:
LOGGER.exception(e)
resp = InternalError(reason=e)
if isinstance(resp, ResponseBase) and 'swift.trans_id' in env:
resp.headers['x-oss-id-2'] = env['swift.trans_id']
resp.headers['x-oss-request-id'] = env['swift.trans_id']
return resp(env, start_response)
def check_filter_order(pipeline, required_filters):
"""
Check that required filters are present in order in the pipeline.
"""
indexes = []
missing_filters = []
for filter in required_filters:
try:
indexes.append(pipeline.index(filter))
except ValueError as e:
LOGGER.debug(e)
missing_filters.append(filter)
if missing_filters:
raise ValueError('Invalid pipeline %r: missing filters %r' %
(pipeline, missing_filters))
if indexes != sorted(indexes):
raise ValueError('Invalid pipeline %r: expected filter %s' %
(pipeline, ' before '.join(required_filters)))
def fromstring(text, root_tag=None):
try:
elem = lxml.etree.fromstring(text, parser)
except lxml.etree.XMLSyntaxError as e:
LOGGER.debug(e)
raise XMLSyntaxError(e)
cleanup_namespaces(elem)
if root_tag is not None:
# validate XML
try:
path = 'schema/%s.rng' % camel_to_snake(root_tag)
with resource_stream(__name__, path) as rng:
lxml.etree.RelaxNG(file=rng).assertValid(elem)
except IOError as e:
# Probably, the schema file doesn't exist.
exc_type, exc_value, exc_traceback = sys.exc_info()
LOGGER.error(e)
raise exc_type, exc_value, exc_traceback
except lxml.etree.DocumentInvalid as e:
LOGGER.debug(e)
raise DocumentInvalid(e)
return elem
def check_pipeline(self, conf):
"""
Check that proxy-server.conf has an appropriate pipeline for 2swift.
"""
if conf.get('__file__', None) is None:
return
ctx = loadcontext(loadwsgi.APP, conf.__file__)
pipeline = str(PipelineWrapper(ctx)).split(' ')
# Add compatible with 3rd party middleware.
check_filter_order(pipeline, ['oss2swift', 'proxy-server'])
auth_pipeline = pipeline[pipeline.index('oss2swift') +
1:pipeline.index('proxy-server')]
# Check SLO middleware
if self.slo_enabled and 'slo' not in auth_pipeline:
self.slo_enabled = False
LOGGER.warning('oss2swift middleware requires SLO middleware '
'to support multi-part upload, please add it '
'in pipeline')
if not conf.auth_pipeline_check:
LOGGER.debug('Skip pipeline auth check.')
return
if 'tempauth' in auth_pipeline:
LOGGER.debug('Use tempauth middleware.')
elif 'keystoneauth' in auth_pipeline:
check_filter_order(auth_pipeline,
['osstoken', 'authtoken', 'keystoneauth'])
LOGGER.debug('Use keystone middleware.')
elif len(auth_pipeline):
LOGGER.debug('Use third party(unknown) auth middleware.')
else:
raise ValueError('Invalid pipeline %r: expected auth between '
'oss2swift and proxy-server ' % pipeline)
def decode_acl(resource, headers, owner):
value = ''
key = sysmeta_header(resource, 'acl')
if key in headers:
value = headers[key]
if value == '':
return ACL(Owner(None, None), [])
try:
id = None
name = None
if owner is not None:
id = owner
name = owner
if id is not None and name is not None:
return ACL(Owner(id, name), value)
except Exception as e:
LOGGER.debug(e)
pass
raise InvalidSubresource((resource, 'acl', value))
def PUT(self, req):
"""
Handle PUT Bucket CoreRule request
"""
xml = req.xml(MAX_PUT_BUCKET_CORERULE_SIZE)
if xml:
# check location
try:
try:
elem = fromstring(xml, 'CORSConfiguration')
except (XMLSyntaxError, DocumentInvalid):
raise InvalidArgument()
for core_rule in elem.findall('CORSRule'):
allowed_origins = _find_all_tags(core_rule,'AllowedOrigin')
allowed_methods = _find_all_tags(core_rule,'AllowedMethod')
allowed_headers= _find_all_tags(core_rule,'AllowedHeader')
expose_headers = _find_all_tags(core_rule,'ExposeHeader')
if core_rule.find('MaxAgeSeconds') is not None:
max_age_seconds = core_rule.find('MaxAgeSeconds').text
req.headers['X-Container-Meta-Access-Control-Allow-Origin'] = _list_str(allowed_origins)
req.headers['X-Container-Meta-Access-Control-Allow-Methods']=_list_str(allowed_methods)
req.headers['X-Container-Meta-Access-Control-Allow-Headers'] = _list_str(allowed_headers)
req.headers['X-Container-Meta-Access-Control-Expose-Headers'] = _list_str(expose_headers)
req.headers['X-Container-Meta-Access-Control-Max-Age'] = max_age_seconds
except (XMLSyntaxError, DocumentInvalid):
raise MalformedXML()
except Exception as e:
exc_type, exc_value, exc_traceback = sys.exc_info()
LOGGER.error(e)
raise exc_type, exc_value, exc_traceback
resp = req.get_response(self.app)
resp.status = HTTP_OK
return resp
def get_acl(headers, body, bucket_owner, object_owner=None):
acl = ACL.from_headers(headers, bucket_owner, object_owner,
as_private=False)
if acl is None:
# Get acl from request body if possible.
if not body:
msg = 'Your request was missing a required header'
raise MissingSecurityHeader(msg, missing_header_name='x-oss-acl')
try:
elem = fromstring(body, ACL.root_tag)
acl = ACL.from_elem(elem)
except(XMLSyntaxError, DocumentInvalid):
raise MalformedACLError()
except Exception as e:
exc_type, exc_value, exc_traceback = sys.exc_info()
LOGGER.error(e)
raise exc_type, exc_value, exc_traceback
else:
if body:
# Specifying grant with both header and xml is not allowed.
raise UnexpectedContent()
return acl
def POST(self, req):
"""
Handles Complete Multipart Upload.
"""
upload_id = req.params['uploadId']
req.headers['x-object-meta-object-type'] = 'Multipart'
resp = _get_upload_info(req, self.app, upload_id)
headers = {}
for key, val in resp.headers.iteritems():
_key = key.lower()
if _key.startswith('x-oss-meta-'):
headers['x-object-meta-' + _key[11:]] = val
elif _key == 'content-type':
headers['Content-Type'] = val
# Query for the objects in the segments area to make sure it completed
query = {
'format': 'json',
'prefix': '%s/%s/' % (req.object_name, upload_id),
'delimiter': '/'
}
container = req.container_name + MULTIUPLOAD_SUFFIX
resp = req.get_response(self.app, 'GET', container, '', query=query)
objinfo = json.loads(resp.body)
objtable = dict((o['name'],
{'path': '/'.join(['', container, o['name']]),
'etag': o['hash'],
'size_bytes': o['bytes']}) for o in objinfo)
manifest = []
previous_number = 0
try:
xml = req.xml(MAX_COMPLETE_UPLOAD_BODY_SIZE)
complete_elem = fromstring(xml, 'CompleteMultipartUpload')
for part_elem in complete_elem.iterchildren('Part'):
part_number = int(part_elem.find('./PartNumber').text)
if part_number <= previous_number:
raise InvalidPartOrder(upload_id=upload_id)
previous_number = part_number
etag = part_elem.find('./ETag').text
if len(etag) >= 2 and etag[0] == '"' and etag[-1] == '"':
# strip double quotes
etag = etag[1:-1]
info = objtable.get("%s/%s/%s" % (req.object_name, upload_id,
part_number))
if info is None or info['etag'] != etag:
raise InvalidPart(upload_id=upload_id,
part_number=part_number)
manifest.append(info)
except (XMLSyntaxError, DocumentInvalid):
raise MalformedXML()
except ErrorResponse:
raise
except Exception as e:
exc_type, exc_value, exc_traceback = sys.exc_info()
LOGGER.error(e)
raise exc_type, exc_value, exc_traceback
# Following swift commit 7f636a5, zero-byte segments aren't allowed,
# even as the final segment
if int(info['size_bytes']) == 0:
manifest.pop()
# Ordinarily, we just let SLO check segment sizes. However, we
# just popped off a zero-byte segment; if there was a second
# zero-byte segment and it was at the end, it would succeed on
# Swift < 2.6.0 and fail on newer Swift. It seems reasonable that
# it should always fail.
if manifest and int(manifest[-1]['size_bytes']) == 0:
raise EntityTooSmall()
try:
# TODO: add support for versioning
if manifest:
resp = req.get_response(self.app, 'PUT',
body=json.dumps(manifest),
query={'multipart-manifest': 'put'},
headers=headers)
else:
# the upload must have consisted of a single zero-length part
# just write it directly
resp = req.get_response(self.app, 'PUT', body='',
headers=headers)
except BadSwiftRequest as e:
msg = str(e)
msg_pre_260 = 'Each segment, except the last, must be at least '
# see https://github.com/openstack/swift/commit/c0866ce
msg_260 = ('too small; each segment, except the last, must be '
'at least ')
# see https://github.com/openstack/swift/commit/7f636a5
msg_post_260 = 'too small; each segment must be at least 1 byte'
if msg.startswith(msg_pre_260) or \
msg_260 in msg or msg_post_260 in msg:
# FIXME: Alibaba OSS allows a smaller object than 5 MB if there is
# only one part. Use a COPY request to copy the part object
# from the segments container instead.
raise EntityTooSmall(msg)
else:
raise
if int(info['size_bytes']) == 0:
# clean up the zero-byte segment
empty_seg_cont, empty_seg_name = info['path'].split('/', 2)[1:]
req.get_response(self.app, 'DELETE',
container=empty_seg_cont, obj=empty_seg_name)
# clean up the multipart-upload record
obj = '%s/%s' % (req.object_name, upload_id)
req.get_response(self.app, 'DELETE', container, obj)
result_elem = Element('CompleteMultipartUploadResult')
# NOTE: boto with sig v4 appends port to HTTP_HOST value at the
# request header when the port is non default value and it makes
# req.host_url like as http://localhost:8080:8080/path
# that obviously invalid. Probably it should be resolved at
# swift.common.swob though, tentatively we are parsing and
# reconstructing the correct host_url info here.
# in detail, https://github.com/boto/boto/pull/3513
parsed_url = urlparse(req.host_url)
host_url = '%s://%s' % (parsed_url.scheme, parsed_url.hostname)
if parsed_url.port:
host_url += ':%s' % parsed_url.port
SubElement(result_elem, 'Location').text = host_url + req.path
SubElement(result_elem, 'Bucket').text = req.container_name
SubElement(result_elem, 'Key').text = req.object_name
SubElement(result_elem, 'ETag').text = resp.etag
resp.body = tostring(result_elem)
resp.status = 200
resp.content_type = "application/xml"
return resp
def POST(self, req):
"""
Handles Delete Multiple Objects.
"""
def object_key_iter(elem):
for obj in elem.iterchildren('Object'):
key = obj.find('./Key').text
if not key:
raise UserKeyMustBeSpecified()
version = obj.find('./VersionId')
if version is not None:
version = version.text
yield key, version
try:
xml = req.xml(MAX_MULTI_DELETE_BODY_SIZE, check_md5=True)
elem = fromstring(xml, 'Delete')
quiet = elem.find('./Quiet')
if quiet is not None and quiet.text.lower() == 'true':
self.quiet = True
else:
self.quiet = False
delete_list = list(object_key_iter(elem))
if len(delete_list) > CONF.max_multi_delete_objects:
raise MalformedXML()
except (XMLSyntaxError, DocumentInvalid):
raise MalformedXML()
except ErrorResponse:
raise
except Exception as e:
exc_type, exc_value, exc_traceback = sys.exc_info()
LOGGER.error(e)
raise exc_type, exc_value, exc_traceback
elem = Element('DeleteResult')
# check bucket existence
try:
req.get_response(self.app, 'HEAD')
except AccessDenied as error:
body = self._gen_error_body(error, elem, delete_list)
return HTTPOk(body=body)
for key, version in delete_list:
if version is not None:
# TODO: delete the specific version of the object
raise OssNotImplemented()
req.object_name = key
try:
query = req.gen_multipart_manifest_delete_query(self.app)
req.get_response(self.app, method='DELETE', query=query)
except NoSuchKey:
pass
except ErrorResponse as e:
error = SubElement(elem, 'Error')
SubElement(error, 'Key').text = key
SubElement(error, 'Code').text = e.__class__.__name__
SubElement(error, 'Message').text = e._msg
continue
if not self.quiet:
deleted = SubElement(elem, 'Deleted')
SubElement(deleted, 'Key').text = key
body = tostring(elem)
return HTTPOk(body=body)
def PUT(self, req):
xml = req.xml(MAX_PUT_BUCKET_BODY_SIZE)
if xml:
# query bucket metadata
resp = req.get_response(self.app, method='GET')
if 'x-oss-meta-rules' in resp.headers:
rules_string = resp.headers['x-oss-meta-rules']
rules_num = len(rules_string.split(','))
if rules_num > MAX_RULE_SIZE:
raise TooManyRules()
else:
rules_string = ''
# get rule from request body
try:
elem = fromstring(xml, 'LifecycleConfiguration')
for r in elem.findall('Rule'):
rule_id = r.find('ID').text
if rule_id in rules_string:
raise RuleIdExisted()
rule_prefix = r.find('Prefix').text
if rule_prefix is None:
rule_prefix = ''
rule_status = r.find('Status').text
expiration = r.find('Expiration')
if expiration.find('Days') is not None:
rule = LifecycleRule(
rule_id,
rule_prefix,
status=rule_status,
expiration=LifecycleExpiration(
days=expiration.find('Days').text))
elif expiration.find('Date') is not None:
rule = LifecycleRule(
rule_id,
rule_prefix,
status=rule_status,
expiration=LifecycleExpiration(
date=expiration.find('Date').text))
else:
raise MalformedXML()
if rules_string == '':
req.headers[
'X-Container-Meta-Rules'] = rule_id + ':' + rule_prefix
else:
req.headers[
'X-Container-Meta-Rules'] = rules_string + ',' + rule_id + ':' + rule_prefix
keys = ("ruleId", "rulePrefix", "ruleStatus", "expireDay",
"createDate")
if rule.expiration.days is not None:
values = (rule.id, rule.prefix, rule.status,
rule.expiration.days, '')
rule_dict = dict(zip(keys, values))
meta_name = 'X-Container-Meta-' + rule.id
req.headers[meta_name] = str(rule_dict)
elif rule.expiration.date is not None:
values = (rule.id, rule.prefix, rule.status, '',
rule.expiration.date)
rule_dict = dict(zip(keys, values))
meta_name = 'X-Container-Meta-' + rule.id
req.headers[meta_name] = str(rule_dict)
else:
pass
except (XMLSyntaxError, DocumentInvalid):
raise MalformedXML()
except Exception as e:
exc_type, exc_value, exc_traceback = sys.exc_info()
LOGGER.error(e)
raise exc_type, exc_value, exc_traceback
resp = req.get_response(self.app, method='POST', headers=req.headers)
resp.Status = HTTP_OK
return resp