def _validate_dates(self): """ Validate Date/X-Amz-Date headers for signature v2 :raises: AccessDenied :raises: RequestTimeTooSkewed """ if self._is_query_auth: self._validate_expire_param() # TODO: make sure the case if timestamp param in query return date_header = self.headers.get('Date') amz_date_header = self.headers.get('X-Amz-Date') if not date_header and not amz_date_header: raise AccessDenied('AWS authentication requires a valid Date ' 'or x-amz-date header') # Anyways, request timestamp should be validated epoch = S3Timestamp(0) if self.timestamp < epoch: raise AccessDenied() # If the standard date is too far ahead or behind, it is an # error delta = 60 * 5 if abs(int(self.timestamp) - int(S3Timestamp.now())) > delta: raise RequestTimeTooSkewed()
def timestamp(self): """ S3Timestamp from Date header. If X-Amz-Date header specified, it will be prior to Date header. :return : S3Timestamp instance """ if not self._timestamp: try: if self._is_query_auth and 'Timestamp' in self.params: # If Timestamp specified in query, it should be prior # to any Date header (is this right?) timestamp = mktime( self.params['Timestamp'], SIGV2_TIMESTAMP_FORMAT) else: timestamp = mktime( self.headers.get('X-Amz-Date', self.headers.get('Date'))) except ValueError: raise AccessDenied('AWS authentication requires a valid Date ' 'or x-amz-date header') if timestamp < 0: raise AccessDenied('AWS authentication requires a valid Date ' 'or x-amz-date header') try: self._timestamp = S3Timestamp(timestamp) except ValueError: # Must be far-future; blame clock skew raise RequestTimeTooSkewed() return self._timestamp
def timestamp(self): """ Return timestamp string according to the auth type The difference from v2 is v4 have to see 'X-Amz-Date' even though it's query auth type. """ if not self._timestamp: try: if self._is_query_auth and 'X-Amz-Date' in self.params: # NOTE(andrey-mp): Date in Signature V4 has different # format timestamp = mktime( self.params['X-Amz-Date'], SIGV4_X_AMZ_DATE_FORMAT) else: if self.headers.get('X-Amz-Date'): timestamp = mktime( self.headers.get('X-Amz-Date'), SIGV4_X_AMZ_DATE_FORMAT) else: timestamp = mktime(self.headers.get('Date')) except (ValueError, TypeError): raise AccessDenied('AWS authentication requires a valid Date ' 'or x-amz-date header') if timestamp < 0: raise AccessDenied('AWS authentication requires a valid Date ' 'or x-amz-date header') try: self._timestamp = S3Timestamp(timestamp) except ValueError: # Must be far-future; blame clock skew raise RequestTimeTooSkewed() return self._timestamp
def _validate_expire_param(self): """ Validate X-Amz-Expires in query parameter :raises: AccessDenied :raises: AuthorizationQueryParametersError :raises: AccessDenined """ err = None try: expires = int(self.params['X-Amz-Expires']) except ValueError: err = 'X-Amz-Expires should be a number' else: if expires < 0: err = 'X-Amz-Expires must be non-negative' elif expires >= 2 ** 63: err = 'X-Amz-Expires should be a number' elif expires > 604800: err = ('X-Amz-Expires must be less than a week (in seconds); ' 'that is, the given X-Amz-Expires must be less than ' '604800 seconds') if err: raise AuthorizationQueryParametersError(err) if int(self.timestamp) + expires < S3Timestamp.now(): raise AccessDenied('Request has expired')
def PUT(self, req): """ Handle PUT Object and PUT Object (Copy) request """ # set X-Timestamp by swift3 to use at copy resp body req_timestamp = S3Timestamp.now() req.headers['X-Timestamp'] = req_timestamp.internal if all(h in req.headers for h in ('X-Amz-Copy-Source', 'X-Amz-Copy-Source-Range')): raise InvalidArgument('x-amz-copy-source-range', req.headers['X-Amz-Copy-Source-Range'], 'Illegal copy header') req.check_copy_source(self.app) resp = req.get_response(self.app) if 'X-Amz-Copy-Source' in req.headers: resp.append_copy_resp_body(req.controller_name, req_timestamp.s3xmlformat) # delete object metadata from response for key in list(resp.headers.keys()): if key.startswith('x-amz-meta-'): del resp.headers[key] resp.status = HTTP_OK return resp
def PUT(self, req): """ Handle PUT Object and PUT Object (Copy) request """ method = 'put-object' # set X-Timestamp by swift3 to use at copy resp body req_timestamp = S3Timestamp.now() req.headers['X-Timestamp'] = req_timestamp.internal if all(h in req.headers for h in ('X-Amz-Copy-Source', 'X-Amz-Copy-Source-Range')): raise InvalidArgument('x-amz-copy-source-range', req.headers['X-Amz-Copy-Source-Range'], 'Illegal copy header') if HTTP_HEADER_TAGGING_KEY in req.headers: tagging = convert_urlquery_to_xml( req.headers.pop(HTTP_HEADER_TAGGING_KEY)) req.headers[OBJECT_TAGGING_HEADER] = tagging req.check_copy_source(self.app) resp = req.get_response(self.app) if 'X-Amz-Copy-Source' in req.headers: method = 'copy-object' resp.append_copy_resp_body(req.controller_name, req_timestamp.s3xmlformat) # delete object metadata from response for key in list(resp.headers.keys()): if key.startswith('x-amz-meta-'): del resp.headers[key] log_s3api_command(req, method) resp.status = HTTP_OK return resp
def GET(self, req): """ Handle GET Service request """ log_s3api_command(req, 'list-buckets') resp = req.get_response(self.app, query={'format': 'json'}) containers = json.loads(resp.body) containers = filter( lambda item: validate_bucket_name(item['name']), containers) # we don't keep the creation time of a bucket (s3cmd doesn't # work without that) so we use something bogus. elem = Element('ListAllMyBucketsResult') owner = SubElement(elem, 'Owner') SubElement(owner, 'ID').text = req.user_id SubElement(owner, 'DisplayName').text = req.user_id buckets = SubElement(elem, 'Buckets') for c in containers: if 'last_modified' in c: ts = last_modified_date_to_timestamp(c['last_modified']) creation_date = S3Timestamp(ts).s3xmlformat else: creation_date = '2009-02-03T16:45:09.000Z' if CONF.s3_acl and CONF.check_bucket_owner: try: cname = c['name'].encode('utf-8') c_resp = req.get_response(self.app, 'HEAD', cname) if 'X-Timestamp' in c_resp.sw_headers: creation_date = S3Timestamp( c_resp.sw_headers['X-Timestamp']).s3xmlformat except AccessDenied: continue except NoSuchBucket: continue bucket = SubElement(buckets, 'Bucket') SubElement(bucket, 'Name').text = c['name'] SubElement(bucket, 'CreationDate').text = creation_date body = tostring(elem) return HTTPOk(content_type='application/xml', body=body)
def _validate_expire_param(self): """ Validate Expires in query parameters :raises: AccessDenied """ # Expires header is a float since epoch try: ex = S3Timestamp(float(self.params['Expires'])) except ValueError: raise AccessDenied() if S3Timestamp.now() > ex: raise AccessDenied('Request has expired') if ex >= 2 ** 31: raise AccessDenied( 'Invalid date (should be seconds since epoch): %s' % self.params['Expires'])
def PUT(self, req): """ Handles Upload Part and Upload Part Copy. """ if 'uploadId' not in req.params: raise InvalidArgument('ResourceType', 'partNumber', 'Unexpected query string parameter') part_number = self.parse_part_number(req) upload_id = req.params['uploadId'] _check_upload_info(req, self.app, upload_id) req.container_name += MULTIUPLOAD_SUFFIX req.object_name = '%s/%s/%d' % (req.object_name, upload_id, part_number) req_timestamp = S3Timestamp.now() req.headers['X-Timestamp'] = req_timestamp.internal source_resp = req.check_copy_source(self.app) if 'X-Amz-Copy-Source' in req.headers and \ 'X-Amz-Copy-Source-Range' in req.headers: rng = req.headers['X-Amz-Copy-Source-Range'] header_valid = True try: rng_obj = Range(rng) if len(rng_obj.ranges) != 1: header_valid = False except ValueError: header_valid = False if not header_valid: err_msg = ('The x-amz-copy-source-range value must be of the ' 'form bytes=first-last where first and last are ' 'the zero-based offsets of the first and last ' 'bytes to copy') raise InvalidArgument('x-amz-source-range', rng, err_msg) source_size = int(source_resp.headers['Content-Length']) if not rng_obj.ranges_for_length(source_size): err_msg = ('Range specified is not valid for source object ' 'of size: %s' % source_size) raise InvalidArgument('x-amz-source-range', rng, err_msg) req.headers['Range'] = rng del req.headers['X-Amz-Copy-Source-Range'] resp = req.get_response(self.app) if 'X-Amz-Copy-Source' in req.headers: resp.append_copy_resp_body(req.controller_name, req_timestamp.s3xmlformat) resp.status = 200 return resp
def test_object_PUT_copy_self_metadata_replace(self): date_header = self.get_date_header() timestamp = mktime(date_header) last_modified = S3Timestamp(timestamp).s3xmlformat header = {'x-amz-metadata-directive': 'REPLACE', 'Date': date_header} status, headers, body = self._test_object_PUT_copy_self( swob.HTTPOk, header, timestamp=timestamp) self.assertEqual(status.split()[0], '200') self.assertEqual(headers['Content-Type'], 'application/xml') self.assertTrue(headers.get('etag') is None) elem = fromstring(body, 'CopyObjectResult') self.assertEqual(elem.find('LastModified').text, last_modified) self.assertEqual(elem.find('ETag').text, '"%s"' % self.etag) _, _, headers = self.swift.calls_with_headers[-1] self.assertEqual(headers['X-Copy-From'], '/bucket/object') self.assertEqual(headers['Content-Length'], '0')
def test_object_PUT_copy(self): date_header = self.get_date_header() timestamp = mktime(date_header) last_modified = S3Timestamp(timestamp).s3xmlformat status, headers, body = self._test_object_PUT_copy( swob.HTTPOk, put_header={'Date': date_header}, timestamp=timestamp) self.assertEqual(status.split()[0], '200') self.assertEqual(headers['Content-Type'], 'application/xml') self.assertTrue(headers.get('etag') is None) self.assertTrue(headers.get('x-amz-meta-something') is None) elem = fromstring(body, 'CopyObjectResult') self.assertEqual(elem.find('LastModified').text, last_modified) self.assertEqual(elem.find('ETag').text, '"%s"' % self.etag) _, _, headers = self.swift.calls_with_headers[-1] self.assertEqual(headers['X-Copy-From'], '/some/source') self.assertEqual(headers['Content-Length'], '0')
def test_object_PUT_copy_no_slash(self): date_header = self.get_date_header() timestamp = mktime(date_header) last_modified = S3Timestamp(timestamp).s3xmlformat # Some clients (like Boto) don't include the leading slash; # AWS seems to tolerate this so we should, too status, headers, body = self._test_object_PUT_copy( swob.HTTPOk, src_path='some/source', put_header={'Date': date_header}, timestamp=timestamp) self.assertEqual(status.split()[0], '200') self.assertEqual(headers['Content-Type'], 'application/xml') self.assertTrue(headers.get('etag') is None) self.assertTrue(headers.get('x-amz-meta-something') is None) elem = fromstring(body, 'CopyObjectResult') self.assertEqual(elem.find('LastModified').text, last_modified) self.assertEqual(elem.find('ETag').text, '"%s"' % self.etag) _, _, headers = self.swift.calls_with_headers[-1] self.assertEqual(headers['X-Copy-From'], '/some/source') self.assertEqual(headers['Content-Length'], '0')
def PUT(self, req): """ Handles Upload Part and Upload Part Copy. """ if 'uploadId' not in req.params: raise InvalidArgument('ResourceType', 'partNumber', 'Unexpected query string parameter') try: part_number = int(req.params['partNumber']) if part_number < 1 or CONF.max_upload_part_num < part_number: raise Exception() except Exception: err_msg = 'Part number must be an integer between 1 and %d,' \ ' inclusive' % CONF.max_upload_part_num raise InvalidArgument('partNumber', req.params['partNumber'], err_msg) upload_id = req.params['uploadId'] _check_upload_info(req, self.app, upload_id) req.container_name += MULTIUPLOAD_SUFFIX req.object_name = '%s/%s/%d' % (req.object_name, upload_id, part_number) req_timestamp = S3Timestamp.now() req.headers['X-Timestamp'] = req_timestamp.internal req.check_copy_source(self.app) resp = req.get_response(self.app) if 'X-Amz-Copy-Source' in req.headers: resp.append_copy_resp_body(req.controller_name, req_timestamp.s3xmlformat) resp.status = 200 return resp
def PUT(self, req): """ Handles Upload Part and Upload Part Copy. """ if 'uploadId' not in req.params: raise InvalidArgument('ResourceType', 'partNumber', 'Unexpected query string parameter') try: part_number = int(req.params['partNumber']) if part_number < 1 or CONF.max_upload_part_num < part_number: raise Exception() except Exception: err_msg = 'Part number must be an integer between 1 and %d,' \ ' inclusive' % CONF.max_upload_part_num raise InvalidArgument('partNumber', req.params['partNumber'], err_msg) upload_id = req.params['uploadId'] _check_upload_info(req, self.app, upload_id) req.container_name += MULTIUPLOAD_SUFFIX req.object_name = '%s/%s/%d' % (req.object_name, upload_id, part_number) req_timestamp = S3Timestamp.now() req.headers['X-Timestamp'] = req_timestamp.internal source_resp = req.check_copy_source(self.app) if 'X-Amz-Copy-Source' in req.headers and \ 'X-Amz-Copy-Source-Range' in req.headers: rng = req.headers['X-Amz-Copy-Source-Range'] header_valid = True try: rng_obj = Range(rng) if len(rng_obj.ranges) != 1: header_valid = False except ValueError: header_valid = False if not header_valid: err_msg = ('The x-amz-copy-source-range value must be of the ' 'form bytes=first-last where first and last are ' 'the zero-based offsets of the first and last ' 'bytes to copy') raise InvalidArgument('x-amz-source-range', rng, err_msg) source_size = int(source_resp.headers['Content-Length']) if not rng_obj.ranges_for_length(source_size): err_msg = ('Range specified is not valid for source object ' 'of size: %s' % source_size) raise InvalidArgument('x-amz-source-range', rng, err_msg) req.headers['Range'] = rng del req.headers['X-Amz-Copy-Source-Range'] resp = req.get_response(self.app) if 'X-Amz-Copy-Source' in req.headers: resp.append_copy_resp_body(req.controller_name, req_timestamp.s3xmlformat) resp.status = 200 return resp
def PUT(self, req): """ Handles Upload Part and Upload Part Copy. """ if 'uploadId' not in req.params: raise InvalidArgument('ResourceType', 'partNumber', 'Unexpected query string parameter') part_number = self.parse_part_number(req) upload_id = req.params['uploadId'] _check_upload_info(req, self.app, upload_id) req.container_name += MULTIUPLOAD_SUFFIX req.object_name = '%s/%s/%d' % (req.object_name, upload_id, part_number) req_timestamp = S3Timestamp.now() req.headers['X-Timestamp'] = req_timestamp.internal source_resp = req.check_copy_source(self.app) method = 'upload-part' if 'X-Amz-Copy-Source' in req.headers and \ 'X-Amz-Copy-Source-Range' in req.headers: rng = req.headers['X-Amz-Copy-Source-Range'] method = 'upload-part-copy' header_valid = True try: rng_obj = Range(rng) if len(rng_obj.ranges) != 1: header_valid = False except ValueError: header_valid = False if not header_valid: err_msg = ('The x-amz-copy-source-range value must be of the ' 'form bytes=first-last where first and last are ' 'the zero-based offsets of the first and last ' 'bytes to copy') raise InvalidArgument('x-amz-source-range', rng, err_msg) source_size = int(source_resp.headers['Content-Length']) if not rng_obj.ranges_for_length(source_size): err_msg = ('Range specified is not valid for source object ' 'of size: %s' % source_size) raise InvalidArgument('x-amz-source-range', rng, err_msg) req.headers['Range'] = rng del req.headers['X-Amz-Copy-Source-Range'] if 'X-Amz-Copy-Source' in req.headers: # Clear some problematic headers that might be on the source req.headers.update({ sysmeta_header('object', 'etag'): '', 'X-Object-Sysmeta-Swift3-Etag': '', # for legacy data 'X-Object-Sysmeta-Slo-Etag': '', 'X-Object-Sysmeta-Slo-Size': '', 'X-Object-Sysmeta-Container-Update-Override-Etag': '', }) log_s3api_command(req, method) resp = req.get_response(self.app) if 'X-Amz-Copy-Source' in req.headers: resp.append_copy_resp_body(req.controller_name, req_timestamp.s3xmlformat) resp.status = 200 return resp