def PUT(self, req):
"""
Handle PUT Object and PUT Object (Copy) request
"""
# set X-Timestamp by oss2swift to use at copy resp body
req_timestamp = OssTimestamp.now()
expireDay = ''
createDate = ''
data = req.body
do_crc64 = crcmod.mkCrcFun(0x142F0E1EBA9EA3693L,
initCrc=0L,
xorOut=0xffffffffffffffffL,
rev=True)
crcValue = do_crc64(data)
req.headers['X-Timestamp'] = req_timestamp.internal
req.headers['x-object-meta-object-type'] = 'Normal'
req.headers['x-object-meta-hash-crc64ecma'] = str(crcValue)
if all(h in req.headers
for h in ('x-oss-copy-source', 'x-oss-copy-source-range')):
raise InvalidArgument('x-oss-copy-source-range',
req.headers['x-oss-copy-source-range'],
'Illegal copy header')
req.check_copy_source(self.app)
bucket_headers = {}
bucket_headers = req.get_container_info(self.app)
expireDay, createDate = self._parse_lifecycle(bucket_headers,
req.object_name)
if expireDay != '':
try:
days = int(expireDay)
expire_sc = str(
int(days * 2 * 3600 + float(req_timestamp.internal)))
req.headers['X-Delete-At'] = expire_sc
except:
raise InvalidArgument('X-Deltete-At', days)
elif createDate != '':
try:
unix_time = to_unixtime(createDate, '%Y-%m-%dT%H:%M:%S.000Z')
if unix_time <= int(req_timestamp):
pass
else:
req.headers['X-Object-Meta-ValidDate'] = unix_time
except:
raise InvalidArgument('X-Object-Meta-ValidDate', createDate)
resp = req.get_response(self.app)
if 'x-oss-copy-source' in req.headers:
resp.append_copy_resp_body(req.controller_name,
req_timestamp.ossxmlformat)
# delete object metadata from response
for key in list(resp.headers.keys()):
if key.startswith('x-oss-meta-'):
del resp.headers[key]
resp.status = HTTP_OK
resp.headers['x-oss-hash-crc64ecma'] = crcValue
return resp
def get_validated_param(self, param, default, limit=MAX_32BIT_INT):
value = default
if param in self.params:
try:
if value < int(self.params[param]):
value = int(self.params[param])
if value < 0:
err_msg = 'Argument %s must be an integer between 0 and' \
' %d' % (param, MAX_32BIT_INT)
raise InvalidArgument(param, self.params[param], err_msg)
if value > MAX_32BIT_INT:
# check the value because int() could build either a long
# instance or a 64bit integer.
raise ValueError()
if limit < value:
value = limit
except ValueError:
err_msg = 'Provided %s not an integer or within ' \
'integer range' % param
raise InvalidArgument(param, self.params[param], err_msg)
return value
def _validate_headers(self):
if 'CONTENT_LENGTH' in self.environ:
try:
if self.content_length < 0:
raise InvalidArgument('Content-Length',
self.content_length)
except (ValueError, TypeError):
raise InvalidArgument('Content-Length',
self.environ['CONTENT_LENGTH'])
self._validate_dates()
if 'Content-MD5' in self.headers:
value = self.headers['Content-MD5']
if not re.match('^[A-Za-z0-9+/]+={0,2}$', value):
# Non-base64-alphabet characters in value.
raise InvalidDigest(content_md5=value)
try:
self.headers['ETag'] = value.decode('base64').encode('hex')
except Exception:
raise InvalidDigest(content_md5=value)
if len(self.headers['ETag']) != 32:
raise InvalidDigest(content_md5=value)
if self.method == 'PUT' and any(
h in self.headers
for h in ('If-Match', 'If-None-Match', 'If-Modified-Since',
'If-Unmodified-Since')):
raise OssNotImplemented(
'Conditional object PUTs are not supported.')
if 'X-Oss-Copy-Source' in self.headers:
try:
check_path_header(self, 'X-Oss-Copy-Source', 2, '')
except swob.HTTPException:
msg = 'Copy Source must mention the source bucket and key: ' \
'sourcebucket/sourcekey'
raise InvalidArgument('x-oss-copy-source',
self.headers['X-Oss-Copy-Source'], msg)
if 'x-oss-metadata-directive' in self.headers:
value = self.headers['x-oss-metadata-directive']
if value not in ('COPY', 'REPLACE'):
err_msg = 'Unknown metadata directive.'
raise InvalidArgument('x-oss-metadata-directive', value,
err_msg)
if 'x-oss-storage-class' in self.headers:
# Only STANDARD is supported now.
if self.headers['x-oss-storage-class'] != 'STANDARD':
raise InvalidStorageClass()
if 'x-oss-mfa' in self.headers:
raise OssNotImplemented('MFA Delete is not supported.')
if 'x-oss-server-side-encryption' in self.headers:
raise OssNotImplemented('Server-side encryption is not supported.')
def to_swift_req(self,
method,
container,
obj,
query=None,
body=None,
headers=None):
sw_req = super(OssAclRequest,
self).to_swift_req(method, container, obj, query, body,
headers)
if self.account:
sw_req.environ['swift_owner'] = True # needed to set ACL
sw_req.environ['swift.authorize_override'] = True
sw_req.environ['swift.authorize'] = lambda req: None
if 'HTTP_X_CONTAINER_SYSMETA_OSS2SWIFT_ACL' in sw_req.environ:
oss_acl = sw_req.environ['HTTP_X_CONTAINER_SYSMETA_OSS2SWIFT_ACL']
if sw_req.query_string:
sw_req.query_string = ''
if oss_acl == '[]':
oss_acl = 'private'
try:
translated_acl = swift_acl_translate(oss_acl)
except ACLError:
raise InvalidArgument('x-oss-acl', oss_acl)
for header, acl in translated_acl:
sw_req.headers[header] = acl
return sw_req
def PUT(self, req):
"""
Handle PUT Bucket Referer request
"""
xml = req.xml(MAX_PUT_BUCKET_REFERER_SIZE)
if xml:
# check referer
try:
elem = fromstring(xml, 'RefererConfiguration')
allow_empyt_referer=elem.find('AllowEmptyReferer').text
if allow_empyt_referer not in ['true','false']:
raise InvalidArgument()
referer_list=elem.find('RefererList')
swift_referers=[]
for referer in referer_list.findall('Referer'):
swift_referers.append(referer.text)
if len(swift_referers)==0 :
req.headers['X-Container-Read']=' '
else:
req.headers['X-Container-Read'] = '.r:'+','.join(get_real_url(swift_referers))
except (XMLSyntaxError, DocumentInvalid):
raise MalformedXML()
except Exception as e:
exc_type, exc_value, exc_traceback = sys.exc_info()
LOGGER.error(e)
raise exc_type, exc_value, exc_traceback
resp = req.get_response(self.app)
resp.status = HTTP_OK
return resp
def handle_acl_header(req):
oss_acl = req.environ['HTTP_X_OSS_ACL']
del req.environ['HTTP_X_OSS_ACL']
if req.query_string:
req.query_string = ''
try:
translated_acl = swift_acl_translate(oss_acl)
except ACLError:
raise InvalidArgument('x-oss-acl', oss_acl)
for header, acl in translated_acl:
req.headers[header] = acl
def PUT(self, req):
"""
Handle PUT Bucket CoreRule request
"""
xml = req.xml(MAX_PUT_BUCKET_CORERULE_SIZE)
if xml:
# check location
try:
try:
elem = fromstring(xml, 'CORSConfiguration')
except (XMLSyntaxError, DocumentInvalid):
raise InvalidArgument()
for core_rule in elem.findall('CORSRule'):
allowed_origins = _find_all_tags(core_rule,'AllowedOrigin')
allowed_methods = _find_all_tags(core_rule,'AllowedMethod')
allowed_headers= _find_all_tags(core_rule,'AllowedHeader')
expose_headers = _find_all_tags(core_rule,'ExposeHeader')
if core_rule.find('MaxAgeSeconds') is not None:
max_age_seconds = core_rule.find('MaxAgeSeconds').text
req.headers['X-Container-Meta-Access-Control-Allow-Origin'] = _list_str(allowed_origins)
req.headers['X-Container-Meta-Access-Control-Allow-Methods']=_list_str(allowed_methods)
req.headers['X-Container-Meta-Access-Control-Allow-Headers'] = _list_str(allowed_headers)
req.headers['X-Container-Meta-Access-Control-Expose-Headers'] = _list_str(expose_headers)
req.headers['X-Container-Meta-Access-Control-Max-Age'] = max_age_seconds
except (XMLSyntaxError, DocumentInvalid):
raise MalformedXML()
except Exception as e:
exc_type, exc_value, exc_traceback = sys.exc_info()
LOGGER.error(e)
raise exc_type, exc_value, exc_traceback
resp = req.get_response(self.app)
resp.status = HTTP_OK
return resp
def PUT(self, req):
"""
Handles Upload Part and Upload Part Copy.
"""
if 'uploadId' not in req.params:
raise InvalidArgument('ResourceType', 'partNumber',
'Unexpected query string parameter')
try:
part_number = int(req.params['partNumber'])
if part_number < 1 or CONF.max_upload_part_num < part_number:
raise Exception()
except Exception:
err_msg = 'Part number must be an integer between 1 and %d,' \
' inclusive' % CONF.max_upload_part_num
raise InvalidArgument('partNumber', req.params['partNumber'],
err_msg)
data = req.body
upload_id = req.params['uploadId']
_check_upload_info(req, self.app, upload_id)
req.container_name += MULTIUPLOAD_SUFFIX
req.object_name = '%s/%s/%d' % (req.object_name, upload_id,
part_number)
req_timestamp = OssTimestamp.now()
req.headers['X-Timestamp'] = req_timestamp.internal
source_resp = req.check_copy_source(self.app)
if 'x-oss-copy-source' in req.headers and \
'x-oss-copy-source-range' in req.headers:
rng = req.headers['x-oss-copy-source-range']
header_valid = True
try:
rng_obj = Range(rng)
if len(rng_obj.ranges) != 1:
header_valid = False
except ValueError:
header_valid = False
if not header_valid:
err_msg = ('The x-oss-copy-source-range value must be of the '
'form bytes=first-last where first and last are '
'the zero-based offsets of the first and last '
'bytes to copy')
raise InvalidArgument('x-oss-source-range', rng, err_msg)
source_size = int(source_resp.headers['Content-Length'])
if not rng_obj.ranges_for_length(source_size):
err_msg = ('Range specified is not valid for source object '
'of size: %s' % source_size)
raise InvalidArgument('x-oss-source-range', rng, err_msg)
req.headers['range'] = rng
del req.headers['x-oss-copy-source-range']
resp = req.get_response(self.app)
do_crc64 = crcmod.mkCrcFun(0x142F0E1EBA9EA3693L, initCrc=0L, xorOut=0xffffffffffffffffL, rev=True)
if 'x-oss-copy-source' in req.headers:
resp.append_copy_resp_body(req.controller_name,
req_timestamp.ossxmlformat)
resp.status = 200
resp.headers['x-oss-hash-crc64ecma']=do_crc64(data)
return resp
def GET(self, req):
"""
Handles List Parts.
"""
def filter_part_num_marker(o):
try:
num = int(os.path.basename(o['name']))
return num > part_num_marker
except ValueError:
return False
encoding_type = req.params.get('encoding-type')
if encoding_type is not None and encoding_type != 'url':
err_msg = 'Invalid Encoding Method specified in Request'
raise InvalidArgument('encoding-type', encoding_type, err_msg)
upload_id = req.params['uploadId']
_check_upload_info(req, self.app, upload_id)
maxparts = req.get_validated_param(
'max-parts', DEFAULT_MAX_PARTS_LISTING, CONF.max_parts_listing)
part_num_marker = req.get_validated_param(
'part-number-marker', 0)
query = {
'format': 'json',
'limit': maxparts + 1,
'prefix': '%s/%s/' % (req.object_name, upload_id),
'delimiter': '/'
}
container = req.container_name + MULTIUPLOAD_SUFFIX
resp = req.get_response(self.app, container=container, obj='',
query=query)
objects = json.loads(resp.body)
last_part = 0
# If the caller requested a list starting at a specific part number,
# construct a sub-set of the object list.
objList = filter(filter_part_num_marker, objects)
# pylint: disable-msg=E1103
objList.sort(key=lambda o: int(o['name'].split('/')[-1]))
if len(objList) > maxparts:
objList = objList[:maxparts]
truncated = True
else:
truncated = False
# TODO: We have to retrieve object list again when truncated is True
# and some objects filtered by invalid name because there could be no
# enough objects for limit defined by maxparts.
if objList:
o = objList[-1]
last_part = os.path.basename(o['name'])
result_elem = Element('ListPartsResult')
SubElement(result_elem, 'Bucket').text = req.container_name
SubElement(result_elem, 'Key').text = req.object_name
SubElement(result_elem, 'UploadId').text = upload_id
initiator_elem = SubElement(result_elem, 'Initiator')
SubElement(initiator_elem, 'ID').text = req.user_id
SubElement(initiator_elem, 'DisplayName').text = req.user_id
owner_elem = SubElement(result_elem, 'Owner')
SubElement(owner_elem, 'ID').text = req.user_id
SubElement(owner_elem, 'DisplayName').text = req.user_id
SubElement(result_elem, 'StorageClass').text = 'STANDARD'
SubElement(result_elem, 'PartNumberMarker').text = str(part_num_marker)
SubElement(result_elem, 'NextPartNumberMarker').text = str(last_part)
SubElement(result_elem, 'MaxParts').text = str(maxparts)
if 'encoding-type' in req.params:
SubElement(result_elem, 'EncodingType').text = \
req.params['encoding-type']
SubElement(result_elem, 'IsTruncated').text = \
'true' if truncated else 'false'
for i in objList:
part_elem = SubElement(result_elem, 'Part')
SubElement(part_elem, 'PartNumber').text = i['name'].split('/')[-1]
SubElement(part_elem, 'LastModified').text = \
i['last_modified'][:-6] + '000Z'
SubElement(part_elem, 'ETag').text = '"%s"' % i['hash']
SubElement(part_elem, 'Size').text = str(i['bytes'])
body = tostring(result_elem, encoding_type=encoding_type)
return HTTPOk(body=body, content_type='application/xml')
def GET(self, req):
"""
Handles List Multipart Uploads
"""
def separate_uploads(uploads, prefix, delimiter):
"""
separate_uploads will separate uploads into non_delimited_uploads
(a subset of uploads) and common_prefixes according to the
specified delimiter. non_delimited_uploads is a list of uploads
which exclude the delimiter. common_prefixes is a set of prefixes
prior to the specified delimiter. Note that the prefix in the
common_prefixes includes the delimiter itself.
i.e. if '/' delimiter specified and then the uploads is consists of
['foo', 'foo/bar'], this function will return (['foo'], ['foo/']).
:param uploads: A list of uploads dictionary
:param prefix: A string of prefix reserved on the upload path.
(i.e. the delimiter must be searched behind the
prefix)
:param delimiter: A string of delimiter to split the path in each
upload
:return (non_delimited_uploads, common_prefixes)
"""
(prefix, delimiter) = \
utf8encode(prefix, delimiter)
non_delimited_uploads = []
common_prefixes = set()
for upload in uploads:
key = upload['key']
end = key.find(delimiter, len(prefix))
if end >= 0:
common_prefix = key[:end + len(delimiter)]
common_prefixes.add(common_prefix)
else:
non_delimited_uploads.append(upload)
return non_delimited_uploads, sorted(common_prefixes)
encoding_type = req.params.get('encoding-type')
if encoding_type is not None and encoding_type != 'url':
err_msg = 'Invalid Encoding Method specified in Request'
raise InvalidArgument('encoding-type', encoding_type, err_msg)
keymarker = req.params.get('key-marker', '')
uploadid = req.params.get('upload-id-marker', '')
maxuploads = req.get_validated_param(
'max-uploads', DEFAULT_MAX_UPLOADS, DEFAULT_MAX_UPLOADS)
query = {
'format': 'json',
'limit': maxuploads + 1,
}
if uploadid and keymarker:
query.update({'marker': '%s/%s' % (keymarker, uploadid)})
elif keymarker:
query.update({'marker': '%s/~' % (keymarker)})
if 'prefix' in req.params:
query.update({'prefix': req.params['prefix']})
container = req.container_name + MULTIUPLOAD_SUFFIX
try:
resp = req.get_response(self.app, container=container, query=query)
objects = json.loads(resp.body)
except NoSuchBucket:
# Assume NoSuchBucket as no uploads
objects = []
def object_to_upload(object_info):
obj, upid = object_info['name'].rsplit('/', 1)
obj_dict = {'key': obj,
'upload_id': upid,
'last_modified': object_info['last_modified']}
return obj_dict
# uploads is a list consists of dict, {key, upload_id, last_modified}
# Note that pattern matcher will drop whole segments objects like as
# object_name/upload_id/1.
pattern = re.compile('/[0-9]+$')
uploads = [object_to_upload(obj) for obj in objects if
pattern.search(obj.get('name', '')) is None]
prefixes = []
if 'delimiter' in req.params:
prefix = req.params.get('prefix', '')
delimiter = req.params['delimiter']
uploads, prefixes = \
separate_uploads(uploads, prefix, delimiter)
if len(uploads) > maxuploads:
uploads = uploads[:maxuploads]
truncated = True
else:
truncated = False
nextkeymarker = ''
nextuploadmarker = ''
if len(uploads) > 1:
nextuploadmarker = uploads[-1]['upload_id']
nextkeymarker = uploads[-1]['key']
result_elem = Element('ListMultipartUploadsResult')
SubElement(result_elem, 'Bucket').text = req.container_name
SubElement(result_elem, 'KeyMarker').text = keymarker
SubElement(result_elem, 'UploadIdMarker').text = uploadid
SubElement(result_elem, 'NextKeyMarker').text = nextkeymarker
SubElement(result_elem, 'NextUploadIdMarker').text = nextuploadmarker
if 'delimiter' in req.params:
SubElement(result_elem, 'Delimiter').text = \
req.params['delimiter']
if 'prefix' in req.params:
SubElement(result_elem, 'Prefix').text = req.params['prefix']
SubElement(result_elem, 'MaxUploads').text = str(maxuploads)
if encoding_type is not None:
SubElement(result_elem, 'EncodingType').text = encoding_type
SubElement(result_elem, 'IsTruncated').text = \
'true' if truncated else 'false'
# TODO: don't show uploads which are initiated before this bucket is
# created.
for u in uploads:
upload_elem = SubElement(result_elem, 'Upload')
SubElement(upload_elem, 'Key').text = u['key']
SubElement(upload_elem, 'UploadId').text = u['upload_id']
initiator_elem = SubElement(upload_elem, 'Initiator')
SubElement(initiator_elem, 'ID').text = req.user_id
SubElement(initiator_elem, 'DisplayName').text = req.user_id
owner_elem = SubElement(upload_elem, 'Owner')
SubElement(owner_elem, 'ID').text = req.user_id
SubElement(owner_elem, 'DisplayName').text = req.user_id
SubElement(upload_elem, 'StorageClass').text = 'STANDARD'
SubElement(upload_elem, 'Initiated').text = \
u['last_modified'][:-6] + '000Z'
for p in prefixes:
elem = SubElement(result_elem, 'CommonPrefixes')
SubElement(elem, 'Prefix').text = p
body = tostring(result_elem, encoding_type=encoding_type)
return HTTPOk(body=body, content_type='application/xml')
def GET(self, req):
"""
Handle GET Bucket (List Objects) request
"""
max_keys = req.get_validated_param('max-keys', CONF.max_bucket_listing)
# TODO: Separate max_bucket_listing and default_bucket_listing
tag_max_keys = max_keys
max_keys = min(max_keys, CONF.max_bucket_listing)
encoding_type = req.params.get('encoding-type')
if encoding_type is not None and encoding_type != 'url':
err_msg = 'Invalid Encoding Method specified in Request'
raise InvalidArgument('encoding-type', encoding_type, err_msg)
query = {
'format': 'json',
'limit': max_keys + 1,
}
if 'marker' in req.params:
query.update({'marker': req.params['marker']})
if 'prefix' in req.params:
query.update({'prefix': req.params['prefix']})
if 'delimiter' in req.params:
query.update({'delimiter': req.params['delimiter']})
resp = req.get_response(self.app, query=query)
if resp.headers['x-oss-website-redirect']:
index = resp.headers['x-oss-web-index']
resp = req.get_response(self.app, obj=index)
return HTTPOk(body=resp.body, content_type='application/xml')
objects = json.loads(resp.body)
elem = Element('ListBucketResult')
SubElement(elem, 'Name').text = req.container_name
SubElement(elem, 'Prefix').text = req.params.get('prefix')
SubElement(elem, 'Marker').text = req.params.get('marker')
# in order to judge that truncated is valid, check whether
# max_keys + 1 th element exists in swift.
is_truncated = max_keys > 0 and len(objects) > max_keys
objects = objects[:max_keys]
if is_truncated and 'delimiter' in req.params:
if 'name' in objects[-1]:
SubElement(elem, 'NextMarker').text = \
objects[-1]['name']
if 'subdir' in objects[-1]:
SubElement(elem, 'NextMarker').text = \
objects[-1]['subdir']
SubElement(elem, 'MaxKeys').text = str(tag_max_keys)
if 'delimiter' in req.params:
SubElement(elem, 'Delimiter').text = req.params['delimiter']
if encoding_type is not None:
SubElement(elem, 'EncodingType').text = encoding_type
SubElement(elem, 'IsTruncated').text = \
'true' if is_truncated else 'false'
for o in objects:
if 'subdir' not in o:
contents = SubElement(elem, 'Contents')
SubElement(contents, 'Key').text = o['name']
SubElement(contents, 'LastModified').text = \
o['last_modified'][:-6] + '000Z'
SubElement(contents, 'ETag').text = '"%s"' % o['hash']
SubElement(contents, 'Size').text = str(o['bytes'])
owner = SubElement(contents, 'Owner')
SubElement(owner, 'ID').text = req.user_id
SubElement(owner, 'DisplayName').text = req.user_id
SubElement(contents, 'StorageClass').text = 'STANDARD'
SubElement(contents, 'Type').text = 'Normal'
for o in objects:
if 'subdir' in o:
common_prefixes = SubElement(elem, 'CommonPrefixes')
SubElement(common_prefixes, 'Prefix').text = o['subdir']
body = tostring(elem, encoding_type=encoding_type)
return HTTPOk(body=body, content_type='application/xml')