def _validate_dates(self):
"""
Validate Date/X-Amz-Date headers for signature v2
:raises: AccessDenied
:raises: RequestTimeTooSkewed
"""
if self._is_query_auth:
self._validate_expire_param()
# TODO: make sure the case if timestamp param in query
return
date_header = self.headers.get('Date')
amz_date_header = self.headers.get('X-Amz-Date')
if not date_header and not amz_date_header:
raise AccessDenied('AWS authentication requires a valid Date '
'or x-amz-date header')
# Anyways, request timestamp should be validated
epoch = S3Timestamp(0)
if self.timestamp < epoch:
raise AccessDenied()
# If the standard date is too far ahead or behind, it is an
# error
delta = 60 * 5
if abs(int(self.timestamp) - int(S3Timestamp.now())) > delta:
raise RequestTimeTooSkewed()
def timestamp(self):
"""
S3Timestamp from Date header. If X-Amz-Date header specified, it
will be prior to Date header.
:return : S3Timestamp instance
"""
if not self._timestamp:
try:
if self._is_query_auth and 'Timestamp' in self.params:
# If Timestamp specified in query, it should be prior
# to any Date header (is this right?)
timestamp = mktime(
self.params['Timestamp'], SIGV2_TIMESTAMP_FORMAT)
else:
timestamp = mktime(
self.headers.get('X-Amz-Date',
self.headers.get('Date')))
except ValueError:
raise AccessDenied('AWS authentication requires a valid Date '
'or x-amz-date header')
if timestamp < 0:
raise AccessDenied('AWS authentication requires a valid Date '
'or x-amz-date header')
try:
self._timestamp = S3Timestamp(timestamp)
except ValueError:
# Must be far-future; blame clock skew
raise RequestTimeTooSkewed()
return self._timestamp
def timestamp(self):
"""
Return timestamp string according to the auth type
The difference from v2 is v4 have to see 'X-Amz-Date' even though
it's query auth type.
"""
if not self._timestamp:
try:
if self._is_query_auth and 'X-Amz-Date' in self.params:
# NOTE(andrey-mp): Date in Signature V4 has different
# format
timestamp = mktime(
self.params['X-Amz-Date'], SIGV4_X_AMZ_DATE_FORMAT)
else:
if self.headers.get('X-Amz-Date'):
timestamp = mktime(
self.headers.get('X-Amz-Date'),
SIGV4_X_AMZ_DATE_FORMAT)
else:
timestamp = mktime(self.headers.get('Date'))
except (ValueError, TypeError):
raise AccessDenied('AWS authentication requires a valid Date '
'or x-amz-date header')
if timestamp < 0:
raise AccessDenied('AWS authentication requires a valid Date '
'or x-amz-date header')
try:
self._timestamp = S3Timestamp(timestamp)
except ValueError:
# Must be far-future; blame clock skew
raise RequestTimeTooSkewed()
return self._timestamp
def _validate_expire_param(self):
"""
Validate X-Amz-Expires in query parameter
:raises: AccessDenied
:raises: AuthorizationQueryParametersError
:raises: AccessDenined
"""
err = None
try:
expires = int(self.params['X-Amz-Expires'])
except ValueError:
err = 'X-Amz-Expires should be a number'
else:
if expires < 0:
err = 'X-Amz-Expires must be non-negative'
elif expires >= 2 ** 63:
err = 'X-Amz-Expires should be a number'
elif expires > 604800:
err = ('X-Amz-Expires must be less than a week (in seconds); '
'that is, the given X-Amz-Expires must be less than '
'604800 seconds')
if err:
raise AuthorizationQueryParametersError(err)
if int(self.timestamp) + expires < S3Timestamp.now():
raise AccessDenied('Request has expired')
def PUT(self, req):
"""
Handle PUT Object and PUT Object (Copy) request
"""
# set X-Timestamp by swift3 to use at copy resp body
req_timestamp = S3Timestamp.now()
req.headers['X-Timestamp'] = req_timestamp.internal
if all(h in req.headers
for h in ('X-Amz-Copy-Source', 'X-Amz-Copy-Source-Range')):
raise InvalidArgument('x-amz-copy-source-range',
req.headers['X-Amz-Copy-Source-Range'],
'Illegal copy header')
req.check_copy_source(self.app)
resp = req.get_response(self.app)
if 'X-Amz-Copy-Source' in req.headers:
resp.append_copy_resp_body(req.controller_name,
req_timestamp.s3xmlformat)
# delete object metadata from response
for key in list(resp.headers.keys()):
if key.startswith('x-amz-meta-'):
del resp.headers[key]
resp.status = HTTP_OK
return resp
def PUT(self, req):
"""
Handle PUT Object and PUT Object (Copy) request
"""
method = 'put-object'
# set X-Timestamp by swift3 to use at copy resp body
req_timestamp = S3Timestamp.now()
req.headers['X-Timestamp'] = req_timestamp.internal
if all(h in req.headers
for h in ('X-Amz-Copy-Source', 'X-Amz-Copy-Source-Range')):
raise InvalidArgument('x-amz-copy-source-range',
req.headers['X-Amz-Copy-Source-Range'],
'Illegal copy header')
if HTTP_HEADER_TAGGING_KEY in req.headers:
tagging = convert_urlquery_to_xml(
req.headers.pop(HTTP_HEADER_TAGGING_KEY))
req.headers[OBJECT_TAGGING_HEADER] = tagging
req.check_copy_source(self.app)
resp = req.get_response(self.app)
if 'X-Amz-Copy-Source' in req.headers:
method = 'copy-object'
resp.append_copy_resp_body(req.controller_name,
req_timestamp.s3xmlformat)
# delete object metadata from response
for key in list(resp.headers.keys()):
if key.startswith('x-amz-meta-'):
del resp.headers[key]
log_s3api_command(req, method)
resp.status = HTTP_OK
return resp
def GET(self, req):
"""
Handle GET Service request
"""
log_s3api_command(req, 'list-buckets')
resp = req.get_response(self.app, query={'format': 'json'})
containers = json.loads(resp.body)
containers = filter(
lambda item: validate_bucket_name(item['name']), containers)
# we don't keep the creation time of a bucket (s3cmd doesn't
# work without that) so we use something bogus.
elem = Element('ListAllMyBucketsResult')
owner = SubElement(elem, 'Owner')
SubElement(owner, 'ID').text = req.user_id
SubElement(owner, 'DisplayName').text = req.user_id
buckets = SubElement(elem, 'Buckets')
for c in containers:
if 'last_modified' in c:
ts = last_modified_date_to_timestamp(c['last_modified'])
creation_date = S3Timestamp(ts).s3xmlformat
else:
creation_date = '2009-02-03T16:45:09.000Z'
if CONF.s3_acl and CONF.check_bucket_owner:
try:
cname = c['name'].encode('utf-8')
c_resp = req.get_response(self.app, 'HEAD', cname)
if 'X-Timestamp' in c_resp.sw_headers:
creation_date = S3Timestamp(
c_resp.sw_headers['X-Timestamp']).s3xmlformat
except AccessDenied:
continue
except NoSuchBucket:
continue
bucket = SubElement(buckets, 'Bucket')
SubElement(bucket, 'Name').text = c['name']
SubElement(bucket, 'CreationDate').text = creation_date
body = tostring(elem)
return HTTPOk(content_type='application/xml', body=body)
def _validate_expire_param(self):
"""
Validate Expires in query parameters
:raises: AccessDenied
"""
# Expires header is a float since epoch
try:
ex = S3Timestamp(float(self.params['Expires']))
except ValueError:
raise AccessDenied()
if S3Timestamp.now() > ex:
raise AccessDenied('Request has expired')
if ex >= 2 ** 31:
raise AccessDenied(
'Invalid date (should be seconds since epoch): %s' %
self.params['Expires'])
def PUT(self, req):
"""
Handles Upload Part and Upload Part Copy.
"""
if 'uploadId' not in req.params:
raise InvalidArgument('ResourceType', 'partNumber',
'Unexpected query string parameter')
part_number = self.parse_part_number(req)
upload_id = req.params['uploadId']
_check_upload_info(req, self.app, upload_id)
req.container_name += MULTIUPLOAD_SUFFIX
req.object_name = '%s/%s/%d' % (req.object_name, upload_id,
part_number)
req_timestamp = S3Timestamp.now()
req.headers['X-Timestamp'] = req_timestamp.internal
source_resp = req.check_copy_source(self.app)
if 'X-Amz-Copy-Source' in req.headers and \
'X-Amz-Copy-Source-Range' in req.headers:
rng = req.headers['X-Amz-Copy-Source-Range']
header_valid = True
try:
rng_obj = Range(rng)
if len(rng_obj.ranges) != 1:
header_valid = False
except ValueError:
header_valid = False
if not header_valid:
err_msg = ('The x-amz-copy-source-range value must be of the '
'form bytes=first-last where first and last are '
'the zero-based offsets of the first and last '
'bytes to copy')
raise InvalidArgument('x-amz-source-range', rng, err_msg)
source_size = int(source_resp.headers['Content-Length'])
if not rng_obj.ranges_for_length(source_size):
err_msg = ('Range specified is not valid for source object '
'of size: %s' % source_size)
raise InvalidArgument('x-amz-source-range', rng, err_msg)
req.headers['Range'] = rng
del req.headers['X-Amz-Copy-Source-Range']
resp = req.get_response(self.app)
if 'X-Amz-Copy-Source' in req.headers:
resp.append_copy_resp_body(req.controller_name,
req_timestamp.s3xmlformat)
resp.status = 200
return resp
def test_object_PUT_copy_self_metadata_replace(self):
date_header = self.get_date_header()
timestamp = mktime(date_header)
last_modified = S3Timestamp(timestamp).s3xmlformat
header = {'x-amz-metadata-directive': 'REPLACE', 'Date': date_header}
status, headers, body = self._test_object_PUT_copy_self(
swob.HTTPOk, header, timestamp=timestamp)
self.assertEqual(status.split()[0], '200')
self.assertEqual(headers['Content-Type'], 'application/xml')
self.assertTrue(headers.get('etag') is None)
elem = fromstring(body, 'CopyObjectResult')
self.assertEqual(elem.find('LastModified').text, last_modified)
self.assertEqual(elem.find('ETag').text, '"%s"' % self.etag)
_, _, headers = self.swift.calls_with_headers[-1]
self.assertEqual(headers['X-Copy-From'], '/bucket/object')
self.assertEqual(headers['Content-Length'], '0')
def test_object_PUT_copy(self):
date_header = self.get_date_header()
timestamp = mktime(date_header)
last_modified = S3Timestamp(timestamp).s3xmlformat
status, headers, body = self._test_object_PUT_copy(
swob.HTTPOk, put_header={'Date': date_header}, timestamp=timestamp)
self.assertEqual(status.split()[0], '200')
self.assertEqual(headers['Content-Type'], 'application/xml')
self.assertTrue(headers.get('etag') is None)
self.assertTrue(headers.get('x-amz-meta-something') is None)
elem = fromstring(body, 'CopyObjectResult')
self.assertEqual(elem.find('LastModified').text, last_modified)
self.assertEqual(elem.find('ETag').text, '"%s"' % self.etag)
_, _, headers = self.swift.calls_with_headers[-1]
self.assertEqual(headers['X-Copy-From'], '/some/source')
self.assertEqual(headers['Content-Length'], '0')
def test_object_PUT_copy_no_slash(self):
date_header = self.get_date_header()
timestamp = mktime(date_header)
last_modified = S3Timestamp(timestamp).s3xmlformat
# Some clients (like Boto) don't include the leading slash;
# AWS seems to tolerate this so we should, too
status, headers, body = self._test_object_PUT_copy(
swob.HTTPOk, src_path='some/source',
put_header={'Date': date_header}, timestamp=timestamp)
self.assertEqual(status.split()[0], '200')
self.assertEqual(headers['Content-Type'], 'application/xml')
self.assertTrue(headers.get('etag') is None)
self.assertTrue(headers.get('x-amz-meta-something') is None)
elem = fromstring(body, 'CopyObjectResult')
self.assertEqual(elem.find('LastModified').text, last_modified)
self.assertEqual(elem.find('ETag').text, '"%s"' % self.etag)
_, _, headers = self.swift.calls_with_headers[-1]
self.assertEqual(headers['X-Copy-From'], '/some/source')
self.assertEqual(headers['Content-Length'], '0')
def PUT(self, req):
"""
Handles Upload Part and Upload Part Copy.
"""
if 'uploadId' not in req.params:
raise InvalidArgument('ResourceType', 'partNumber',
'Unexpected query string parameter')
try:
part_number = int(req.params['partNumber'])
if part_number < 1 or CONF.max_upload_part_num < part_number:
raise Exception()
except Exception:
err_msg = 'Part number must be an integer between 1 and %d,' \
' inclusive' % CONF.max_upload_part_num
raise InvalidArgument('partNumber', req.params['partNumber'],
err_msg)
upload_id = req.params['uploadId']
_check_upload_info(req, self.app, upload_id)
req.container_name += MULTIUPLOAD_SUFFIX
req.object_name = '%s/%s/%d' % (req.object_name, upload_id,
part_number)
req_timestamp = S3Timestamp.now()
req.headers['X-Timestamp'] = req_timestamp.internal
req.check_copy_source(self.app)
resp = req.get_response(self.app)
if 'X-Amz-Copy-Source' in req.headers:
resp.append_copy_resp_body(req.controller_name,
req_timestamp.s3xmlformat)
resp.status = 200
return resp
def PUT(self, req):
"""
Handles Upload Part and Upload Part Copy.
"""
if 'uploadId' not in req.params:
raise InvalidArgument('ResourceType', 'partNumber',
'Unexpected query string parameter')
try:
part_number = int(req.params['partNumber'])
if part_number < 1 or CONF.max_upload_part_num < part_number:
raise Exception()
except Exception:
err_msg = 'Part number must be an integer between 1 and %d,' \
' inclusive' % CONF.max_upload_part_num
raise InvalidArgument('partNumber', req.params['partNumber'],
err_msg)
upload_id = req.params['uploadId']
_check_upload_info(req, self.app, upload_id)
req.container_name += MULTIUPLOAD_SUFFIX
req.object_name = '%s/%s/%d' % (req.object_name, upload_id,
part_number)
req_timestamp = S3Timestamp.now()
req.headers['X-Timestamp'] = req_timestamp.internal
req.check_copy_source(self.app)
resp = req.get_response(self.app)
if 'X-Amz-Copy-Source' in req.headers:
resp.append_copy_resp_body(req.controller_name,
req_timestamp.s3xmlformat)
resp.status = 200
return resp
def PUT(self, req):
"""
Handles Upload Part and Upload Part Copy.
"""
if 'uploadId' not in req.params:
raise InvalidArgument('ResourceType', 'partNumber',
'Unexpected query string parameter')
try:
part_number = int(req.params['partNumber'])
if part_number < 1 or CONF.max_upload_part_num < part_number:
raise Exception()
except Exception:
err_msg = 'Part number must be an integer between 1 and %d,' \
' inclusive' % CONF.max_upload_part_num
raise InvalidArgument('partNumber', req.params['partNumber'],
err_msg)
upload_id = req.params['uploadId']
_check_upload_info(req, self.app, upload_id)
req.container_name += MULTIUPLOAD_SUFFIX
req.object_name = '%s/%s/%d' % (req.object_name, upload_id,
part_number)
req_timestamp = S3Timestamp.now()
req.headers['X-Timestamp'] = req_timestamp.internal
source_resp = req.check_copy_source(self.app)
if 'X-Amz-Copy-Source' in req.headers and \
'X-Amz-Copy-Source-Range' in req.headers:
rng = req.headers['X-Amz-Copy-Source-Range']
header_valid = True
try:
rng_obj = Range(rng)
if len(rng_obj.ranges) != 1:
header_valid = False
except ValueError:
header_valid = False
if not header_valid:
err_msg = ('The x-amz-copy-source-range value must be of the '
'form bytes=first-last where first and last are '
'the zero-based offsets of the first and last '
'bytes to copy')
raise InvalidArgument('x-amz-source-range', rng, err_msg)
source_size = int(source_resp.headers['Content-Length'])
if not rng_obj.ranges_for_length(source_size):
err_msg = ('Range specified is not valid for source object '
'of size: %s' % source_size)
raise InvalidArgument('x-amz-source-range', rng, err_msg)
req.headers['Range'] = rng
del req.headers['X-Amz-Copy-Source-Range']
resp = req.get_response(self.app)
if 'X-Amz-Copy-Source' in req.headers:
resp.append_copy_resp_body(req.controller_name,
req_timestamp.s3xmlformat)
resp.status = 200
return resp
def PUT(self, req):
"""
Handles Upload Part and Upload Part Copy.
"""
if 'uploadId' not in req.params:
raise InvalidArgument('ResourceType', 'partNumber',
'Unexpected query string parameter')
part_number = self.parse_part_number(req)
upload_id = req.params['uploadId']
_check_upload_info(req, self.app, upload_id)
req.container_name += MULTIUPLOAD_SUFFIX
req.object_name = '%s/%s/%d' % (req.object_name, upload_id,
part_number)
req_timestamp = S3Timestamp.now()
req.headers['X-Timestamp'] = req_timestamp.internal
source_resp = req.check_copy_source(self.app)
method = 'upload-part'
if 'X-Amz-Copy-Source' in req.headers and \
'X-Amz-Copy-Source-Range' in req.headers:
rng = req.headers['X-Amz-Copy-Source-Range']
method = 'upload-part-copy'
header_valid = True
try:
rng_obj = Range(rng)
if len(rng_obj.ranges) != 1:
header_valid = False
except ValueError:
header_valid = False
if not header_valid:
err_msg = ('The x-amz-copy-source-range value must be of the '
'form bytes=first-last where first and last are '
'the zero-based offsets of the first and last '
'bytes to copy')
raise InvalidArgument('x-amz-source-range', rng, err_msg)
source_size = int(source_resp.headers['Content-Length'])
if not rng_obj.ranges_for_length(source_size):
err_msg = ('Range specified is not valid for source object '
'of size: %s' % source_size)
raise InvalidArgument('x-amz-source-range', rng, err_msg)
req.headers['Range'] = rng
del req.headers['X-Amz-Copy-Source-Range']
if 'X-Amz-Copy-Source' in req.headers:
# Clear some problematic headers that might be on the source
req.headers.update({
sysmeta_header('object', 'etag'):
'',
'X-Object-Sysmeta-Swift3-Etag':
'', # for legacy data
'X-Object-Sysmeta-Slo-Etag':
'',
'X-Object-Sysmeta-Slo-Size':
'',
'X-Object-Sysmeta-Container-Update-Override-Etag':
'',
})
log_s3api_command(req, method)
resp = req.get_response(self.app)
if 'X-Amz-Copy-Source' in req.headers:
resp.append_copy_resp_body(req.controller_name,
req_timestamp.s3xmlformat)
resp.status = 200
return resp
