def PUT(self, req): """ Handles Upload Part and Upload Part Copy. """ if 'uploadId' not in req.params: raise InvalidArgument('ResourceType', 'partNumber', 'Unexpected query string parameter') upload_id = req.params['uploadId'] try: # TODO: check the range of partNumber part_number = int(req.params['partNumber']) except Exception: err_msg = 'Part number must be an integer' raise InvalidArgument('partNumber', req.params['partNumber'], err_msg) _check_upload_info(req, self.app, upload_id) req.container_name += '+segments' req.object_name = '%s/%s/%d' % (req.object_name, upload_id, part_number) resp = req.get_response(self.app) # TODO: set xml body for copy requests. resp.status = 200 return resp
def PUT(self, req): """ Handles Upload Part and Upload Part Copy. """ if 'uploadId' not in req.params: raise InvalidArgument('ResourceType', 'partNumber', 'Unexpected query string parameter') try: part_number = int(req.params['partNumber']) if part_number < 1 or CONF.max_upload_part_num < part_number: raise Exception() except Exception: err_msg = 'Part number must be an integer between 1 and %d,' \ ' inclusive' % CONF.max_upload_part_num raise InvalidArgument('partNumber', req.params['partNumber'], err_msg) upload_id = req.params['uploadId'] _check_upload_info(req, self.app, upload_id) req.container_name += MULTIUPLOAD_SUFFIX req.object_name = '%s/%s/%d' % (req.object_name, upload_id, part_number) req.check_copy_source(self.app) resp = req.get_response(self.app) if 'X-Amz-Copy-Source' in req.headers: resp.append_copy_resp_body(req.controller_name) resp.status = 200 return resp
def PUT(self, req): """ Handles Upload Part and Upload Part Copy. """ if 'uploadId' not in req.params: raise InvalidArgument('ResourceType', 'partNumber', 'Unexpected query string parameter') part_number = self.parse_part_number(req) upload_id = req.params['uploadId'] _check_upload_info(req, self.app, upload_id) req.container_name += MULTIUPLOAD_SUFFIX req.object_name = '%s/%s/%d' % (req.object_name, upload_id, part_number) req_timestamp = S3Timestamp.now() req.headers['X-Timestamp'] = req_timestamp.internal source_resp = req.check_copy_source(self.app) if 'X-Amz-Copy-Source' in req.headers and \ 'X-Amz-Copy-Source-Range' in req.headers: rng = req.headers['X-Amz-Copy-Source-Range'] header_valid = True try: rng_obj = Range(rng) if len(rng_obj.ranges) != 1: header_valid = False except ValueError: header_valid = False if not header_valid: err_msg = ('The x-amz-copy-source-range value must be of the ' 'form bytes=first-last where first and last are ' 'the zero-based offsets of the first and last ' 'bytes to copy') raise InvalidArgument('x-amz-source-range', rng, err_msg) source_size = int(source_resp.headers['Content-Length']) if not rng_obj.ranges_for_length(source_size): err_msg = ('Range specified is not valid for source object ' 'of size: %s' % source_size) raise InvalidArgument('x-amz-source-range', rng, err_msg) req.headers['Range'] = rng del req.headers['X-Amz-Copy-Source-Range'] resp = req.get_response(self.app) if 'X-Amz-Copy-Source' in req.headers: resp.append_copy_resp_body(req.controller_name, req_timestamp.s3xmlformat) resp.status = 200 return resp
def PUT(self, req): """ Handle PUT Object and PUT Object (Copy) request """ method = 'put-object' # set X-Timestamp by swift3 to use at copy resp body req_timestamp = S3Timestamp.now() req.headers['X-Timestamp'] = req_timestamp.internal if all(h in req.headers for h in ('X-Amz-Copy-Source', 'X-Amz-Copy-Source-Range')): raise InvalidArgument('x-amz-copy-source-range', req.headers['X-Amz-Copy-Source-Range'], 'Illegal copy header') if HTTP_HEADER_TAGGING_KEY in req.headers: tagging = convert_urlquery_to_xml( req.headers.pop(HTTP_HEADER_TAGGING_KEY)) req.headers[OBJECT_TAGGING_HEADER] = tagging req.check_copy_source(self.app) resp = req.get_response(self.app) if 'X-Amz-Copy-Source' in req.headers: method = 'copy-object' resp.append_copy_resp_body(req.controller_name, req_timestamp.s3xmlformat) # delete object metadata from response for key in list(resp.headers.keys()): if key.startswith('x-amz-meta-'): del resp.headers[key] log_s3api_command(req, method) resp.status = HTTP_OK return resp
def handle_acl_header(req): """ Handle the x-amz-acl header. """ # Used this method, delete 'HTTP_X_AMZ_ACL' from environ, and header for # s3_acl(x-container-sysmeta-swift3-acl) becomes impossible to create. # TODO: Modify to be able to use the s3_acl and swift acl # (e.g. X-Container-Read) at the same time, if s3_acl is effective. if CONF.s3_acl: return amz_acl = req.environ['HTTP_X_AMZ_ACL'] # Translate the Amazon ACL to something that can be # implemented in Swift, 501 otherwise. Swift uses POST # for ACLs, whereas S3 uses PUT. del req.environ['HTTP_X_AMZ_ACL'] if req.query_string: req.query_string = '' try: translated_acl = swift_acl_translate(amz_acl) except ACLError: raise InvalidArgument('x-amz-acl', amz_acl) for header, acl in translated_acl: req.headers[header] = acl
def _parse_query_authentication(self): """ Parse v4 query authentication - version 4: 'X-Amz-Credential' and 'X-Amz-Signature' should be in param :raises: AccessDenied :raises: AuthorizationHeaderMalformed """ if self.params.get('X-Amz-Algorithm') != 'AWS4-HMAC-SHA256': raise InvalidArgument('X-Amz-Algorithm', self.params.get('X-Amz-Algorithm')) try: cred_param = self.params['X-Amz-Credential'].split("/") access = cred_param[0] sig = self.params['X-Amz-Signature'] expires = self.params['X-Amz-Expires'] except KeyError: raise AccessDenied() try: signed_headers = self.params['X-Amz-SignedHeaders'] except KeyError: # TODO: make sure if is it malformed request? raise AuthorizationHeaderMalformed() self._signed_headers = set(signed_headers.split(';')) # credential must be in following format: # <access-key-id>/<date>/<AWS-region>/<AWS-service>/aws4_request if not all([access, sig, len(cred_param) == 5, expires]): raise AccessDenied() return access, sig
def _parse_authorization(self): if 'AWSAccessKeyId' in self.params: try: self.headers['Date'] = self.params['Expires'] self.headers['Authorization'] = \ 'AWS %(AWSAccessKeyId)s:%(Signature)s' % self.params except KeyError: raise AccessDenied() if 'Authorization' not in self.headers: raise NotS3Request() try: keyword, info = self.headers['Authorization'].split(' ', 1) except Exception: raise AccessDenied() if keyword != 'AWS': raise NotS3Request() try: access_key, signature = info.rsplit(':', 1) except Exception: err_msg = 'AWS authorization header is invalid. ' \ 'Expected AwsAccessKeyId:signature' raise InvalidArgument('Authorization', self.headers['Authorization'], err_msg) return access_key, signature
def PUT(self, req): """ Handle PUT Object and PUT Object (Copy) request """ # set X-Timestamp by swift3 to use at copy resp body req_timestamp = S3Timestamp.now() req.headers['X-Timestamp'] = req_timestamp.internal if all(h in req.headers for h in ('X-Amz-Copy-Source', 'X-Amz-Copy-Source-Range')): raise InvalidArgument('x-amz-copy-source-range', req.headers['X-Amz-Copy-Source-Range'], 'Illegal copy header') req.check_copy_source(self.app) resp = req.get_response(self.app) if 'X-Amz-Copy-Source' in req.headers: resp.append_copy_resp_body(req.controller_name, req_timestamp.s3xmlformat) # delete object metadata from response for key in list(resp.headers.keys()): if key.startswith('x-amz-meta-'): del resp.headers[key] resp.status = HTTP_OK return resp
def get_group_subclass_from_uri(uri): """ Convert a URI to one of the predefined groups. """ for group in Group.__subclasses__(): # pylint: disable-msg=E1101 if group.uri == uri: return group raise InvalidArgument('uri', uri, 'Invalid group uri')
def get_validated_param(self, param, default, limit=None): value = default if param in self.params: try: value = int(self.params[param]) if value < 0 or (limit is not None and limit < value): err_msg = 'Argument %s must be an integer between 0 and' \ ' %d' % (param, limit) raise InvalidArgument(param, self.params[param], err_msg) if not isinstance(value, int): # check the instance because int() could build # a long instance raise ValueError() except ValueError: err_msg = 'Provided %s not an integer or within ' \ 'integer range' % param raise InvalidArgument(param, self.params[param], err_msg) return value
def check_copy_source(self, app): """ check_copy_source checks the copy source existence and if copying an object to itself, for illegal request parameters :returns: the source HEAD response """ try: src_path = self.headers['X-Amz-Copy-Source'] except KeyError: return None if '?' in src_path: src_path, qs = src_path.split('?', 1) query = parse_qsl(qs, True) if not query: pass # ignore it elif len(query) > 1 or query[0][0] != 'versionId': raise InvalidArgument('X-Amz-Copy-Source', self.headers['X-Amz-Copy-Source'], 'Unsupported copy source parameter.') elif query[0][1] != 'null': # TODO: once we support versioning, we'll need to translate # src_path to the proper location in the versions container raise S3NotImplemented('Versioning is not yet supported') self.headers['X-Amz-Copy-Source'] = src_path src_path = unquote(src_path) src_path = src_path if src_path.startswith('/') else ('/' + src_path) src_bucket, src_obj = split_path(src_path, 0, 2, True) headers = swob.HeaderKeyDict() headers.update(self._copy_source_headers()) src_resp = self.get_response(app, 'HEAD', src_bucket, src_obj, headers=headers) if src_resp.status_int == 304: # pylint: disable-msg=E1101 raise PreconditionFailed() self.headers['X-Amz-Copy-Source'] = \ '/' + self.headers['X-Amz-Copy-Source'].lstrip('/') source_container, source_obj = \ split_path(self.headers['X-Amz-Copy-Source'], 1, 2, True) if (self.container_name == source_container and self.object_name == source_obj and self.headers.get('x-amz-metadata-directive', 'COPY') == 'COPY'): raise InvalidRequest("This copy request is illegal " "because it is trying to copy an " "object to itself without " "changing the object's metadata, " "storage class, website redirect " "location or encryption " "attributes.") return src_resp
def from_headers(cls, headers, bucket_owner, object_owner=None, as_private=True, inherit_grants=None): """ Convert HTTP headers to an ACL instance. """ grants = [] if inherit_grants: grants += inherit_grants try: for key, value in headers.items(): if key.lower().startswith('x-amz-grant-'): permission = key[len('x-amz-grant-'):] permission = permission.upper().replace('-', '_') if permission not in PERMISSIONS: continue for grantee in value.split(','): grants.append( Grant(Grantee.from_header(grantee), permission)) if 'x-amz-acl' in headers: try: acl = headers['x-amz-acl'] if len(grants) > 0: err_msg = 'Specifying both Canned ACLs and Header ' \ 'Grants is not allowed' raise InvalidRequest(err_msg) # skip empty ACL headers sent by clients if len(acl): grantees = canned_acl_grantees(bucket_owner, object_owner)[acl] for permission, grantee in grantees: grants.append(Grant(grantee, permission)) except KeyError: # expects canned_acl_grantees()[] raises KeyError raise InvalidArgument('x-amz-acl', headers['x-amz-acl']) except (KeyError, ValueError): # TODO: think about we really catch this except sequence raise InvalidRequest() if len(grants) == 0: # No ACL headers if as_private: return ACLPrivate(bucket_owner, object_owner) else: return None return cls(object_owner or bucket_owner, grants)
def get_validated_param(self, param, default, limit=MAX_32BIT_INT): value = default if param in self.params: try: value = int(self.params[param]) if value < 0: err_msg = 'Argument %s must be an integer between 0 and' \ ' %d' % (param, MAX_32BIT_INT) raise InvalidArgument(param, self.params[param], err_msg) if value > MAX_32BIT_INT: # check the value because int() could build either a long # instance or a 64bit integer. raise ValueError() if limit < value: value = limit except ValueError: err_msg = 'Provided %s not an integer or within ' \ 'integer range' % param raise InvalidArgument(param, self.params[param], err_msg) return value
def convert_urlquery_to_xml(val): """Convert x-amz-tagging to a Tagging XML.""" root = Element('Tagging') elem = SubElement(root, 'TagSet') # AWS support key1=&key2= items = parse_qs(val, keep_blank_values=True) for key, val in items.items(): if len(val) != 1: raise InvalidArgument(HTTP_HEADER_TAGGING_KEY, value=val, msg=INVALID_TAGGING) tag = SubElement(elem, 'Tag') SubElement(tag, 'Key').text = key SubElement(tag, 'Value').text = val[0] return tostring(root)
def parse_part_number(self, req): """ Parse the part number from query string. Raise InvalidArgument if missing or invalid. """ try: part_number = int(req.params['partNumber']) if part_number < 1 or CONF.max_upload_part_num < part_number: raise Exception() except Exception: err_msg = 'Part number must be an integer between 1 and %d,' \ ' inclusive' % CONF.max_upload_part_num raise InvalidArgument('partNumber', req.params['partNumber'], err_msg) return part_number
def from_header(grantee): """ Convert a grantee string in the HTTP header to an Grantee instance. """ type, value = grantee.split('=', 1) value = value.strip('"\'') if type == 'id': return User(value) elif type == 'emailAddress': raise S3NotImplemented() elif type == 'uri': # return a subclass instance of Group class subclass = get_group_subclass_from_uri(value) return subclass() else: raise InvalidArgument(type, value, 'Argument format not recognized')
def handle_acl_header(req): """ Handle the x-amz-acl header. """ amz_acl = req.environ['HTTP_X_AMZ_ACL'] # Translate the Amazon ACL to something that can be # implemented in Swift, 501 otherwise. Swift uses POST # for ACLs, whereas S3 uses PUT. del req.environ['HTTP_X_AMZ_ACL'] if req.query_string: req.query_string = '' try: translated_acl = swift_acl_translate(amz_acl) except ACLError: raise InvalidArgument('x-amz-acl', amz_acl) for header, acl in translated_acl: req.headers[header] = acl
def handle_acl_header(req): """ Handle the x-amz-acl header. Note that this header currently used for only normal-acl (not implemented) on s3acl. TODO: add translation to swift acl like as x-container-read to s3acl """ amz_acl = req.environ['HTTP_X_AMZ_ACL'] # Translate the Amazon ACL to something that can be # implemented in Swift, 501 otherwise. Swift uses POST # for ACLs, whereas S3 uses PUT. del req.environ['HTTP_X_AMZ_ACL'] if req.query_string: req.query_string = '' try: translated_acl = swift_acl_translate(amz_acl) except ACLError: raise InvalidArgument('x-amz-acl', amz_acl) for header, acl in translated_acl: req.headers[header] = acl
def _validate_headers(self): if 'CONTENT_LENGTH' in self.environ: try: if self.content_length < 0: raise InvalidArgument('Content-Length', self.content_length) except (ValueError, TypeError): raise InvalidArgument('Content-Length', self.environ['CONTENT_LENGTH']) self._validate_dates() value = _header_strip(self.headers.get('Content-MD5')) if value is not None: if not re.match('^[A-Za-z0-9+/]+={0,2}$', value): # Non-base64-alphabet characters in value. raise InvalidDigest(content_md5=value) try: self.headers['ETag'] = value.decode('base64').encode('hex') except Exception: raise InvalidDigest(content_md5=value) if len(self.headers['ETag']) != 32: raise InvalidDigest(content_md5=value) if self.method == 'PUT' and any(h in self.headers for h in ( 'If-Match', 'If-None-Match', 'If-Modified-Since', 'If-Unmodified-Since')): raise S3NotImplemented( 'Conditional object PUTs are not supported.') if 'X-Amz-Copy-Source' in self.headers: try: check_path_header(self, 'X-Amz-Copy-Source', 2, '') except swob.HTTPException: msg = 'Copy Source must mention the source bucket and key: ' \ 'sourcebucket/sourcekey' raise InvalidArgument('x-amz-copy-source', self.headers['X-Amz-Copy-Source'], msg) if 'x-amz-metadata-directive' in self.headers: value = self.headers['x-amz-metadata-directive'] if value not in ('COPY', 'REPLACE'): err_msg = 'Unknown metadata directive.' raise InvalidArgument('x-amz-metadata-directive', value, err_msg) if 'x-amz-storage-class' in self.headers: # Only STANDARD is supported now. if self.headers['x-amz-storage-class'] != 'STANDARD': raise InvalidStorageClass() if 'x-amz-mfa' in self.headers: raise S3NotImplemented('MFA Delete is not supported.') if 'x-amz-server-side-encryption' in self.headers: raise S3NotImplemented('Server-side encryption is not supported.') if 'x-amz-website-redirect-location' in self.headers: raise S3NotImplemented('Website redirection is not supported.')
def PUT(self, req): """ Handles Upload Part and Upload Part Copy. """ if 'uploadId' not in req.params: raise InvalidArgument('ResourceType', 'partNumber', 'Unexpected query string parameter') part_number = self.parse_part_number(req) upload_id = req.params['uploadId'] _check_upload_info(req, self.app, upload_id) req.container_name += MULTIUPLOAD_SUFFIX req.object_name = '%s/%s/%d' % (req.object_name, upload_id, part_number) req_timestamp = S3Timestamp.now() req.headers['X-Timestamp'] = req_timestamp.internal source_resp = req.check_copy_source(self.app) method = 'upload-part' if 'X-Amz-Copy-Source' in req.headers and \ 'X-Amz-Copy-Source-Range' in req.headers: rng = req.headers['X-Amz-Copy-Source-Range'] method = 'upload-part-copy' header_valid = True try: rng_obj = Range(rng) if len(rng_obj.ranges) != 1: header_valid = False except ValueError: header_valid = False if not header_valid: err_msg = ('The x-amz-copy-source-range value must be of the ' 'form bytes=first-last where first and last are ' 'the zero-based offsets of the first and last ' 'bytes to copy') raise InvalidArgument('x-amz-source-range', rng, err_msg) source_size = int(source_resp.headers['Content-Length']) if not rng_obj.ranges_for_length(source_size): err_msg = ('Range specified is not valid for source object ' 'of size: %s' % source_size) raise InvalidArgument('x-amz-source-range', rng, err_msg) req.headers['Range'] = rng del req.headers['X-Amz-Copy-Source-Range'] if 'X-Amz-Copy-Source' in req.headers: # Clear some problematic headers that might be on the source req.headers.update({ sysmeta_header('object', 'etag'): '', 'X-Object-Sysmeta-Swift3-Etag': '', # for legacy data 'X-Object-Sysmeta-Slo-Etag': '', 'X-Object-Sysmeta-Slo-Size': '', 'X-Object-Sysmeta-Container-Update-Override-Etag': '', }) log_s3api_command(req, method) resp = req.get_response(self.app) if 'X-Amz-Copy-Source' in req.headers: resp.append_copy_resp_body(req.controller_name, req_timestamp.s3xmlformat) resp.status = 200 return resp
def GET(self, req): """ Handle GET Bucket (List Objects) request """ max_keys = req.get_validated_param('max-keys', CONF.max_bucket_listing) # TODO: Separate max_bucket_listing and default_bucket_listing tag_max_keys = max_keys max_keys = min(max_keys, CONF.max_bucket_listing) encoding_type = req.params.get('encoding-type') if encoding_type is not None and encoding_type != 'url': err_msg = 'Invalid Encoding Method specified in Request' raise InvalidArgument('encoding-type', encoding_type, err_msg) query = { 'format': 'json', 'limit': max_keys + 1, } if 'marker' in req.params: query.update({'marker': req.params['marker']}) if 'prefix' in req.params: query.update({'prefix': req.params['prefix']}) if 'delimiter' in req.params: query.update({'delimiter': req.params['delimiter']}) # GET Bucket (List Objects) Version 2 parameters is_v2 = int(req.params.get('list-type', '1')) == 2 fetch_owner = False if is_v2: if 'start-after' in req.params: query.update({'marker': req.params['start-after']}) # continuation-token overrides start-after if 'continuation-token' in req.params: decoded = b64decode(req.params['continuation-token']) query.update({'marker': decoded}) if 'fetch-owner' in req.params: fetch_owner = config_true_value(req.params['fetch-owner']) resp = req.get_response(self.app, query=query) objects = json.loads(resp.body) if 'versions' in req.params: req.container_name += VERSIONING_SUFFIX query['reverse'] = 'true' try: resp = req.get_response(self.app, query=query) versioned_objects = json.loads(resp.body) prefixes = set() for o in versioned_objects: if 'name' in o: # The name looks like this: # '%03x%s/%s' % (len(name), name, version) o['name'], o['version_id'] = \ o['name'][3:].rsplit('/', 1) else: prefixes.add(o['subdir']) # suppress duplicated prefixes for o in list(objects): if 'subdir' in o and o['subdir'] in prefixes: objects.remove(o) objects.extend(versioned_objects) except NoSuchBucket: # the bucket may not be versioned pass req.container_name = req.container_name[:-len(VERSIONING_SUFFIX)] objects.sort(key=lambda o: o.get('name') or o.get('subdir')) for o in objects: if 'subdir' not in o and not o.get('version_id'): info = req.get_object_info( self.app, object_name=o['name'].encode('utf-8')) o['sysmeta_version_id'] = info.get('sysmeta', {}).get( 'version-id', 'null') if 'versions' in req.params: elem = Element('ListVersionsResult') else: elem = Element('ListBucketResult') if encoding_type is not None: elem.encoding_type = encoding_type SubElement(elem, 'Name').text = req.container_name SubElement(elem, 'Prefix').text = req.params.get('prefix') # Filter objects according to version-id-marker and key-marker v_marker = req.params.get('version-id-marker') k_marker = req.params.get('key-marker') k_marker_matched = not bool(k_marker) if 'versions' in req.params and (v_marker or k_marker): to_delete = [] for i, o in enumerate(objects): if 'subdir' not in o: version_id = o.get('version_id', o.get('sysmeta_version_id', 'null')) if not k_marker_matched and k_marker != o['name']: to_delete.append(i) if k_marker == o['name']: k_marker_matched = True if k_marker == o['name'] and v_marker: if v_marker == version_id: v_marker = None to_delete.append(i) for i in reversed(to_delete): objects.pop(i) # in order to judge that truncated is valid, check whether # max_keys + 1 th element exists in swift. is_truncated = max_keys > 0 and len(objects) > max_keys objects = objects[:max_keys] if not is_v2: if 'versions' in req.params: SubElement(elem, 'KeyMarker').text = req.params.get('key-marker') SubElement(elem, 'VersionIdMarker').text = req.params.get( 'version-id-marker') else: SubElement(elem, 'Marker').text = req.params.get('marker') if is_truncated and 'delimiter' in req.params: if 'name' in objects[-1]: SubElement(elem, 'NextMarker').text = \ objects[-1]['name'] if 'subdir' in objects[-1]: SubElement(elem, 'NextMarker').text = \ objects[-1]['subdir'] else: if is_truncated: if 'name' in objects[-1]: SubElement(elem, 'NextContinuationToken').text = \ b64encode(objects[-1]['name'].encode('utf8')) if 'subdir' in objects[-1]: SubElement(elem, 'NextContinuationToken').text = \ b64encode(objects[-1]['subdir'].encode('utf8')) if 'continuation-token' in req.params: SubElement(elem, 'ContinuationToken').text = \ req.params['continuation-token'] if 'start-after' in req.params: SubElement(elem, 'StartAfter').text = \ req.params['start-after'] SubElement(elem, 'KeyCount').text = str(len(objects)) SubElement(elem, 'MaxKeys').text = str(tag_max_keys) if 'delimiter' in req.params: SubElement(elem, 'Delimiter').text = req.params['delimiter'] if encoding_type is not None: SubElement(elem, 'EncodingType').text = encoding_type SubElement(elem, 'IsTruncated').text = \ 'true' if is_truncated else 'false' for o in objects: if 'subdir' not in o: if 'versions' in req.params: version_id = o.get('version_id', o.get('sysmeta_version_id', 'null')) if o.get('content_type') == DELETE_MARKER_CONTENT_TYPE: contents = SubElement(elem, 'DeleteMarker') else: contents = SubElement(elem, 'Version') SubElement(contents, 'Key').text = \ o['name'].encode('utf-8') SubElement(contents, 'VersionId').text = version_id SubElement( contents, 'IsLatest').text = str('version_id' not in o).lower() else: contents = SubElement(elem, 'Contents') SubElement(contents, 'Key').text = \ o['name'].encode('utf-8') SubElement(contents, 'LastModified').text = \ o['last_modified'][:-3] + 'Z' if 's3_etag' in o.get('content_type', ''): _, o['hash'] = extract_s3_etag(o['content_type']) if contents.tag != 'DeleteMarker': SubElement(contents, 'ETag').text = '"%s"' % o['hash'] SubElement(contents, 'Size').text = str(o['bytes']) if fetch_owner or not is_v2: owner = SubElement(contents, 'Owner') SubElement(owner, 'ID').text = req.user_id SubElement(owner, 'DisplayName').text = req.user_id if contents.tag != 'DeleteMarker': SubElement(contents, 'StorageClass').text = 'STANDARD' for o in objects: if 'subdir' in o: common_prefixes = SubElement(elem, 'CommonPrefixes') SubElement(common_prefixes, 'Prefix').text = \ o['subdir'].encode('utf-8') body = tostring(elem, encoding_type=encoding_type) resp = HTTPOk(body=body, content_type='application/xml') origin = req.headers.get('Origin') if origin: rule = get_cors(self.app, req, "GET", origin) if rule: cors_fill_headers(req, resp, rule) return resp
def GET(self, req): """ Handles List Parts. """ encoding_type = req.params.get('encoding-type') if encoding_type is not None and encoding_type != 'url': err_msg = 'Invalid Encoding Method specified in Request' raise InvalidArgument('encoding-type', encoding_type, err_msg) upload_id = req.params['uploadId'] _check_upload_info(req, self.app, upload_id) part_num_marker = 0 # TODO: add support for max-parts and part-number-marker queries. query = { 'format': 'json', 'prefix': '%s/%s/' % (req.object_name, upload_id), 'delimiter': '/' } container = req.container_name + '+segments' resp = req.get_response(self.app, container=container, obj='', query=query) objects = loads(resp.body) last_part = 0 # pylint: disable-msg=E1103 objects.sort(key=lambda o: int(o['name'].split('/')[-1])) if len(objects) > 0: o = objects[-1] last_part = os.path.basename(o['name']) result_elem = Element('ListPartsResult') SubElement(result_elem, 'Bucket').text = req.container_name SubElement(result_elem, 'Key').text = req.object_name SubElement(result_elem, 'UploadId').text = upload_id initiator_elem = SubElement(result_elem, 'Initiator') SubElement(initiator_elem, 'ID').text = req.user_id SubElement(initiator_elem, 'DisplayName').text = req.user_id owner_elem = SubElement(result_elem, 'Owner') SubElement(owner_elem, 'ID').text = req.user_id SubElement(owner_elem, 'DisplayName').text = req.user_id SubElement(result_elem, 'StorageClass').text = 'STANDARD' SubElement(result_elem, 'PartNumberMarker').text = str(part_num_marker) SubElement(result_elem, 'NextPartNumberMarker').text = str(last_part) SubElement(result_elem, 'MaxParts').text = str(DEFAULT_MAX_PARTS) # TODO: add support for EncodingType SubElement(result_elem, 'IsTruncated').text = 'false' for i in objects: part_elem = SubElement(result_elem, 'Part') SubElement(part_elem, 'PartNumber').text = i['name'].split('/')[-1] SubElement(part_elem, 'LastModified').text = \ i['last_modified'][:-3] + 'Z' SubElement(part_elem, 'ETag').text = i['hash'] SubElement(part_elem, 'Size').text = str(i['bytes']) body = tostring(result_elem, encoding_type=encoding_type) return HTTPOk(body=body, content_type='application/xml')
def GET(self, req): """ Handles List Multipart Uploads """ encoding_type = req.params.get('encoding-type') if encoding_type is not None and encoding_type != 'url': err_msg = 'Invalid Encoding Method specified in Request' raise InvalidArgument('encoding-type', encoding_type, err_msg) # TODO: add support for prefix, key-marker, upload-id-marker, and # max-uploads queries. query = { 'format': 'json', } container = req.container_name + '+segments' resp = req.get_response(self.app, container=container, query=query) objects = loads(resp.body) uploads = [] for o in objects: obj, upid = split_path('/' + o['name'], 1, 2, True) if '/' in upid: # This is a part object. continue uploads.append({ 'key': obj, 'upload_id': upid, 'last_modified': o['last_modified'] }) nextkeymarker = '' nextuploadmarker = '' if len(uploads) > 1: nextuploadmarker = uploads[-1]['upload_id'] nextkeymarker = uploads[-1]['key'] result_elem = Element('ListMultipartUploadsResult') SubElement(result_elem, 'Bucket').text = req.container_name SubElement(result_elem, 'KeyMarker').text = '' SubElement(result_elem, 'UploadIdMarker').text = '' SubElement(result_elem, 'NextKeyMarker').text = nextkeymarker SubElement(result_elem, 'NextUploadIdMarker').text = nextuploadmarker SubElement(result_elem, 'MaxUploads').text = str(DEFAULT_MAX_UPLOADS) if encoding_type is not None: SubElement(result_elem, 'EncodingType').text = encoding_type SubElement(result_elem, 'IsTruncated').text = 'false' # TODO: don't show uploads which are initiated before this bucket is # created. for u in uploads: upload_elem = SubElement(result_elem, 'Upload') SubElement(upload_elem, 'Key').text = u['key'] SubElement(upload_elem, 'UploadId').text = u['upload_id'] initiator_elem = SubElement(upload_elem, 'Initiator') SubElement(initiator_elem, 'ID').text = req.user_id SubElement(initiator_elem, 'DisplayName').text = req.user_id owner_elem = SubElement(upload_elem, 'Owner') SubElement(owner_elem, 'ID').text = req.user_id SubElement(owner_elem, 'DisplayName').text = req.user_id SubElement(upload_elem, 'StorageClass').text = 'STANDARD' SubElement(upload_elem, 'Initiated').text = \ u['last_modified'][:-3] + 'Z' body = tostring(result_elem, encoding_type=encoding_type) return HTTPOk(body=body, content_type='application/xml')
def GET(self, req): """ Handles List Parts. """ def filter_part_num_marker(o): try: num = int(os.path.basename(o['name'])) return num > part_num_marker except ValueError: return False encoding_type = req.params.get('encoding-type') if encoding_type is not None and encoding_type != 'url': err_msg = 'Invalid Encoding Method specified in Request' raise InvalidArgument('encoding-type', encoding_type, err_msg) upload_id = req.params['uploadId'] _check_upload_info(req, self.app, upload_id) maxparts = req.get_validated_param('max-parts', DEFAULT_MAX_PARTS_LISTING, CONF.max_parts_listing) part_num_marker = req.get_validated_param('part-number-marker', 0) query = { 'format': 'json', 'limit': maxparts + 1, 'prefix': '%s/%s/' % (req.object_name, upload_id), 'delimiter': '/' } container = req.container_name + MULTIUPLOAD_SUFFIX resp = req.get_response(self.app, container=container, obj='', query=query) objects = json.loads(resp.body) last_part = 0 # If the caller requested a list starting at a specific part number, # construct a sub-set of the object list. objList = filter(filter_part_num_marker, objects) # pylint: disable-msg=E1103 objList.sort(key=lambda o: int(o['name'].split('/')[-1])) if len(objList) > maxparts: objList = objList[:maxparts] truncated = True else: truncated = False # TODO: We have to retrieve object list again when truncated is True # and some objects filtered by invalid name because there could be no # enough objects for limit defined by maxparts. if objList: o = objList[-1] last_part = os.path.basename(o['name']) result_elem = Element('ListPartsResult') SubElement(result_elem, 'Bucket').text = req.container_name SubElement(result_elem, 'Key').text = req.object_name SubElement(result_elem, 'UploadId').text = upload_id initiator_elem = SubElement(result_elem, 'Initiator') SubElement(initiator_elem, 'ID').text = req.user_id SubElement(initiator_elem, 'DisplayName').text = req.user_id owner_elem = SubElement(result_elem, 'Owner') SubElement(owner_elem, 'ID').text = req.user_id SubElement(owner_elem, 'DisplayName').text = req.user_id SubElement(result_elem, 'StorageClass').text = 'STANDARD' SubElement(result_elem, 'PartNumberMarker').text = str(part_num_marker) SubElement(result_elem, 'NextPartNumberMarker').text = str(last_part) SubElement(result_elem, 'MaxParts').text = str(maxparts) if 'encoding-type' in req.params: SubElement(result_elem, 'EncodingType').text = \ req.params['encoding-type'] SubElement(result_elem, 'IsTruncated').text = \ 'true' if truncated else 'false' for i in objList: part_elem = SubElement(result_elem, 'Part') SubElement(part_elem, 'PartNumber').text = i['name'].split('/')[-1] SubElement(part_elem, 'LastModified').text = \ i['last_modified'][:-3] + 'Z' SubElement(part_elem, 'ETag').text = '"%s"' % i['hash'] SubElement(part_elem, 'Size').text = str(i['bytes']) body = tostring(result_elem, encoding_type=encoding_type) return HTTPOk(body=body, content_type='application/xml')
def GET(self, req): """ Handles List Multipart Uploads """ def separate_uploads(uploads, prefix, delimiter): """ separate_uploads will separate uploads into non_delimited_uploads (a subset of uploads) and common_prefixes according to the specified delimiter. non_delimited_uploads is a list of uploads which exclude the delimiter. common_prefixes is a set of prefixes prior to the specified delimiter. Note that the prefix in the common_prefixes includes the delimiter itself. i.e. if '/' delimiter specified and then the uploads is consists of ['foo', 'foo/bar'], this function will return (['foo'], ['foo/']). :param uploads: A list of uploads dictionary :param prefix: A string of prefix reserved on the upload path. (i.e. the delimiter must be searched behind the prefix) :param delimiter: A string of delimiter to split the path in each upload :return (non_delimited_uploads, common_prefixes) """ (prefix, delimiter) = \ utf8encode(prefix, delimiter) non_delimited_uploads = [] common_prefixes = set() for upload in uploads: key = upload['key'] end = key.find(delimiter, len(prefix)) if end >= 0: common_prefix = key[:end + len(delimiter)] common_prefixes.add(common_prefix) else: non_delimited_uploads.append(upload) return non_delimited_uploads, sorted(common_prefixes) encoding_type = req.params.get('encoding-type') if encoding_type is not None and encoding_type != 'url': err_msg = 'Invalid Encoding Method specified in Request' raise InvalidArgument('encoding-type', encoding_type, err_msg) keymarker = req.params.get('key-marker', '') uploadid = req.params.get('upload-id-marker', '') maxuploads = req.get_validated_param('max-uploads', DEFAULT_MAX_UPLOADS, DEFAULT_MAX_UPLOADS) query = { 'format': 'json', 'limit': maxuploads + 1, } if uploadid and keymarker: query.update({'marker': '%s/%s' % (keymarker, uploadid)}) elif keymarker: query.update({'marker': '%s/~' % (keymarker)}) if 'prefix' in req.params: query.update({'prefix': req.params['prefix']}) container = req.container_name + MULTIUPLOAD_SUFFIX try: resp = req.get_response(self.app, container=container, query=query) objects = json.loads(resp.body) except NoSuchBucket: # Assume NoSuchBucket as no uploads objects = [] def object_to_upload(object_info): obj, upid = object_info['name'].rsplit('/', 1) obj_dict = { 'key': obj, 'upload_id': upid, 'last_modified': object_info['last_modified'] } return obj_dict # uploads is a list consists of dict, {key, upload_id, last_modified} # Note that pattern matcher will drop whole segments objects like as # object_name/upload_id/1. pattern = re.compile('/[0-9]+$') uploads = [ object_to_upload(obj) for obj in objects if pattern.search(obj.get('name', '')) is None ] prefixes = [] if 'delimiter' in req.params: prefix = req.params.get('prefix', '') delimiter = req.params['delimiter'] uploads, prefixes = \ separate_uploads(uploads, prefix, delimiter) if len(uploads) > maxuploads: uploads = uploads[:maxuploads] truncated = True else: truncated = False nextkeymarker = '' nextuploadmarker = '' if len(uploads) > 1: nextuploadmarker = uploads[-1]['upload_id'] nextkeymarker = uploads[-1]['key'] result_elem = Element('ListMultipartUploadsResult') SubElement(result_elem, 'Bucket').text = req.container_name SubElement(result_elem, 'KeyMarker').text = keymarker SubElement(result_elem, 'UploadIdMarker').text = uploadid SubElement(result_elem, 'NextKeyMarker').text = nextkeymarker SubElement(result_elem, 'NextUploadIdMarker').text = nextuploadmarker if 'delimiter' in req.params: SubElement(result_elem, 'Delimiter').text = \ req.params['delimiter'] if 'prefix' in req.params: SubElement(result_elem, 'Prefix').text = req.params['prefix'] SubElement(result_elem, 'MaxUploads').text = str(maxuploads) if encoding_type is not None: SubElement(result_elem, 'EncodingType').text = encoding_type SubElement(result_elem, 'IsTruncated').text = \ 'true' if truncated else 'false' # TODO: don't show uploads which are initiated before this bucket is # created. for u in uploads: upload_elem = SubElement(result_elem, 'Upload') SubElement(upload_elem, 'Key').text = u['key'] SubElement(upload_elem, 'UploadId').text = u['upload_id'] initiator_elem = SubElement(upload_elem, 'Initiator') SubElement(initiator_elem, 'ID').text = req.user_id SubElement(initiator_elem, 'DisplayName').text = req.user_id owner_elem = SubElement(upload_elem, 'Owner') SubElement(owner_elem, 'ID').text = req.user_id SubElement(owner_elem, 'DisplayName').text = req.user_id SubElement(upload_elem, 'StorageClass').text = 'STANDARD' SubElement(upload_elem, 'Initiated').text = \ u['last_modified'][:-3] + 'Z' for p in prefixes: elem = SubElement(result_elem, 'CommonPrefixes') SubElement(elem, 'Prefix').text = p body = tostring(result_elem, encoding_type=encoding_type) return HTTPOk(body=body, content_type='application/xml')
def _validate_headers(self): if 'CONTENT_LENGTH' in self.environ: try: if self.content_length < 0: raise InvalidArgument('Content-Length', self.content_length) except (ValueError, TypeError): raise InvalidArgument('Content-Length', self.environ['CONTENT_LENGTH']) if 'Date' in self.headers: now = datetime.datetime.utcnow() date = email.utils.parsedate(self.headers['Date']) if 'Expires' in self.params: try: d = email.utils.formatdate(float(self.params['Expires'])) except ValueError: raise AccessDenied() # check expiration expdate = email.utils.parsedate(d) ex = datetime.datetime(*expdate[0:6]) if now > ex: raise AccessDenied('Request has expired') elif date is not None: epoch = datetime.datetime(1970, 1, 1, 0, 0, 0, 0) d1 = datetime.datetime(*date[0:6]) if d1 < epoch: raise AccessDenied() # If the standard date is too far ahead or behind, it is an # error delta = datetime.timedelta(seconds=60 * 5) if abs(d1 - now) > delta: raise RequestTimeTooSkewed() else: raise AccessDenied() if 'Content-MD5' in self.headers: value = self.headers['Content-MD5'] if not re.match('^[A-Za-z0-9+/]+={0,2}$', value): # Non-base64-alphabet characters in value. raise InvalidDigest(content_md5=value) try: self.headers['ETag'] = value.decode('base64').encode('hex') except Exception: raise InvalidDigest(content_md5=value) if 'X-Amz-Copy-Source' in self.headers: try: check_path_header(self, 'X-Amz-Copy-Source', 2, '') except swob.HTTPException: msg = 'Copy Source must mention the source bucket and key: ' \ 'sourcebucket/sourcekey' raise InvalidArgument('x-amz-copy-source', self.headers['X-Amz-Copy-Source'], msg) if 'x-amz-metadata-directive' in self.headers: value = self.headers['x-amz-metadata-directive'] if value not in ('COPY', 'REPLACE'): err_msg = 'Unknown metadata directive.' raise InvalidArgument('x-amz-metadata-directive', value, err_msg) if 'x-amz-storage-class' in self.headers: # Only STANDARD is supported now. if self.headers['x-amz-storage-class'] != 'STANDARD': raise InvalidStorageClass() if 'x-amz-mfa' in self.headers: raise S3NotImplemented('MFA Delete is not supported.') if 'x-amz-server-side-encryption' in self.headers: raise S3NotImplemented('Server-side encryption is not supported.') if 'x-amz-website-redirect-location' in self.headers: raise S3NotImplemented('Website redirection is not supported.')
def GET(self, req): """ Handle GET Bucket (List Objects) request """ max_keys = req.get_validated_param('max-keys', CONF.max_bucket_listing) # TODO: Separate max_bucket_listing and default_bucket_listing tag_max_keys = max_keys max_keys = min(max_keys, CONF.max_bucket_listing) encoding_type = req.params.get('encoding-type') if encoding_type is not None and encoding_type != 'url': err_msg = 'Invalid Encoding Method specified in Request' raise InvalidArgument('encoding-type', encoding_type, err_msg) query = { 'format': 'json', 'limit': max_keys + 1, } if 'marker' in req.params: query.update({'marker': req.params['marker']}) if 'prefix' in req.params: query.update({'prefix': req.params['prefix']}) if 'delimiter' in req.params: query.update({'delimiter': req.params['delimiter']}) resp = req.get_response(self.app, query=query) objects = json.loads(resp.body) elem = Element('ListBucketResult') SubElement(elem, 'Name').text = req.container_name SubElement(elem, 'Prefix').text = req.params.get('prefix') SubElement(elem, 'Marker').text = req.params.get('marker') # in order to judge that truncated is valid, check whether # max_keys + 1 th element exists in swift. is_truncated = max_keys > 0 and len(objects) > max_keys objects = objects[:max_keys] if is_truncated and 'delimiter' in req.params: if 'name' in objects[-1]: SubElement(elem, 'NextMarker').text = \ objects[-1]['name'] if 'subdir' in objects[-1]: SubElement(elem, 'NextMarker').text = \ objects[-1]['subdir'] SubElement(elem, 'MaxKeys').text = str(tag_max_keys) if 'delimiter' in req.params: SubElement(elem, 'Delimiter').text = req.params['delimiter'] if encoding_type is not None: SubElement(elem, 'EncodingType').text = encoding_type SubElement(elem, 'IsTruncated').text = \ 'true' if is_truncated else 'false' for o in objects: if 'subdir' not in o: contents = SubElement(elem, 'Contents') SubElement(contents, 'Key').text = o['name'] SubElement(contents, 'LastModified').text = \ o['last_modified'][:-3] + 'Z' SubElement(contents, 'ETag').text = '"%s"' % o['hash'] SubElement(contents, 'Size').text = str(o['bytes']) owner = SubElement(contents, 'Owner') SubElement(owner, 'ID').text = req.user_id SubElement(owner, 'DisplayName').text = req.user_id SubElement(contents, 'StorageClass').text = 'STANDARD' for o in objects: if 'subdir' in o: common_prefixes = SubElement(elem, 'CommonPrefixes') SubElement(common_prefixes, 'Prefix').text = o['subdir'] body = tostring(elem, encoding_type=encoding_type) return HTTPOk(body=body, content_type='application/xml')
def GET(self, req): """ Handle GET Bucket (List Objects) request """ if 'max-keys' in req.params: if req.params.get('max-keys').isdigit() is False: raise InvalidArgument('max-keys', req.params['max-keys']) max_keys = int(req.params.get('max-keys', CONF.max_bucket_listing)) max_keys = min(max_keys, CONF.max_bucket_listing) encoding_type = req.params.get('encoding-type') if encoding_type is not None and encoding_type != 'url': err_msg = 'Invalid Encoding Method specified in Request' raise InvalidArgument('encoding-type', encoding_type, err_msg) query = { 'format': 'json', 'limit': max_keys + 1, } if 'marker' in req.params: query.update({'marker': req.params['marker']}) if 'prefix' in req.params: query.update({'prefix': req.params['prefix']}) if 'delimiter' in req.params: query.update({'delimiter': req.params['delimiter']}) resp = req.get_response(self.app, query=query) objects = loads(resp.body) elem = Element('ListBucketResult') SubElement(elem, 'Name').text = req.container_name SubElement(elem, 'Prefix').text = req.params.get('prefix') SubElement(elem, 'Marker').text = req.params.get('marker') SubElement(elem, 'MaxKeys').text = str(max_keys) if 'delimiter' in req.params: SubElement(elem, 'Delimiter').text = req.params['delimiter'] if encoding_type is not None: SubElement(elem, 'EncodingType').text = encoding_type if max_keys > 0 and len(objects) == max_keys + 1: is_truncated = 'true' else: is_truncated = 'false' SubElement(elem, 'IsTruncated').text = is_truncated for o in objects[:max_keys]: if 'subdir' not in o: contents = SubElement(elem, 'Contents') SubElement(contents, 'Key').text = o['name'] SubElement(contents, 'LastModified').text = \ o['last_modified'] + 'Z' SubElement(contents, 'ETag').text = o['hash'] SubElement(contents, 'Size').text = str(o['bytes']) owner = SubElement(contents, 'Owner') SubElement(owner, 'ID').text = req.user_id SubElement(owner, 'DisplayName').text = req.user_id SubElement(contents, 'StorageClass').text = 'STANDARD' for o in objects[:max_keys]: if 'subdir' in o: common_prefixes = SubElement(elem, 'CommonPrefixes') SubElement(common_prefixes, 'Prefix').text = o['subdir'] body = tostring(elem, encoding_type=encoding_type) return HTTPOk(body=body, content_type='application/xml')
def GET(self, req): """ Handles List Multipart Uploads """ def filter_max_uploads(o): name = o.get('name', '') return name.count('/') == 1 encoding_type = req.params.get('encoding-type') if encoding_type is not None and encoding_type != 'url': err_msg = 'Invalid Encoding Method specified in Request' raise InvalidArgument('encoding-type', encoding_type, err_msg) # TODO: add support for delimiter query. keymarker = req.params.get('key-marker', '') uploadid = req.params.get('upload-id-marker', '') maxuploads = req.get_validated_param('max-uploads', DEFAULT_MAX_UPLOADS, DEFAULT_MAX_UPLOADS) query = { 'format': 'json', 'limit': maxuploads + 1, } if uploadid and keymarker: query.update({'marker': '%s/%s' % (keymarker, uploadid)}) elif keymarker: query.update({'marker': '%s/~' % (keymarker)}) if 'prefix' in req.params: query.update({'prefix': req.params['prefix']}) container = req.container_name + MULTIUPLOAD_SUFFIX resp = req.get_response(self.app, container=container, query=query) objects = json.loads(resp.body) objects = filter(filter_max_uploads, objects) if len(objects) > maxuploads: objects = objects[:maxuploads] truncated = True else: truncated = False uploads = [] prefixes = [] for o in objects: obj, upid = split_path('/' + o['name'], 1, 2) uploads.append({ 'key': obj, 'upload_id': upid, 'last_modified': o['last_modified'] }) nextkeymarker = '' nextuploadmarker = '' if len(uploads) > 1: nextuploadmarker = uploads[-1]['upload_id'] nextkeymarker = uploads[-1]['key'] result_elem = Element('ListMultipartUploadsResult') SubElement(result_elem, 'Bucket').text = req.container_name SubElement(result_elem, 'KeyMarker').text = keymarker SubElement(result_elem, 'UploadIdMarker').text = uploadid SubElement(result_elem, 'NextKeyMarker').text = nextkeymarker SubElement(result_elem, 'NextUploadIdMarker').text = nextuploadmarker if 'prefix' in req.params: SubElement(result_elem, 'Prefix').text = req.params['prefix'] SubElement(result_elem, 'MaxUploads').text = str(maxuploads) if encoding_type is not None: SubElement(result_elem, 'EncodingType').text = encoding_type SubElement(result_elem, 'IsTruncated').text = \ 'true' if truncated else 'false' # TODO: don't show uploads which are initiated before this bucket is # created. for u in uploads: upload_elem = SubElement(result_elem, 'Upload') SubElement(upload_elem, 'Key').text = u['key'] SubElement(upload_elem, 'UploadId').text = u['upload_id'] initiator_elem = SubElement(upload_elem, 'Initiator') SubElement(initiator_elem, 'ID').text = req.user_id SubElement(initiator_elem, 'DisplayName').text = req.user_id owner_elem = SubElement(upload_elem, 'Owner') SubElement(owner_elem, 'ID').text = req.user_id SubElement(owner_elem, 'DisplayName').text = req.user_id SubElement(upload_elem, 'StorageClass').text = 'STANDARD' SubElement(upload_elem, 'Initiated').text = \ u['last_modified'][:-3] + 'Z' for p in prefixes: elem = SubElement(result_elem, 'CommonPrefixes') SubElement(elem, 'Prefix').text = p body = tostring(result_elem, encoding_type=encoding_type) return HTTPOk(body=body, content_type='application/xml')