def PUT(self, req): """ Handles PUT Bucket acl and PUT Object acl. """ log_s3api_command(req, 'put-acl') if req.is_object_request: # Handle Object ACL raise S3NotImplemented() else: # Handle Bucket ACL xml = req.xml(MAX_ACL_BODY_SIZE) if all(['HTTP_X_AMZ_ACL' in req.environ, xml]): # S3 doesn't allow to give ACL with both ACL header and body. raise UnexpectedContent() elif not any(['HTTP_X_AMZ_ACL' in req.environ, xml]): # Both canned ACL header and xml body are missing raise MissingSecurityHeader(missing_header_name='x-amz-acl') else: # correct ACL exists in the request if xml: # We very likely have an XML-based ACL request. # let's try to translate to the request header try: translated_acl = swift_acl_translate(xml, xml=True) except ACLError: raise MalformedACLError() for header, acl in translated_acl: req.headers[header] = acl resp = req.get_response(self.app, 'POST') resp.status = HTTP_OK resp.headers.update({'Location': req.container_name}) return resp
def PUT(self, req): """ Handles PUT Bucket versioning. """ log_s3api_command(req, 'put-bucket-versioning') xml = req.xml(MAX_PUT_VERSIONING_BODY_SIZE) try: elem = fromstring(xml, 'VersioningConfiguration') status = elem.find('./Status').text except (XMLSyntaxError, DocumentInvalid): raise MalformedXML() except Exception as e: LOGGER.error(e) raise if status not in ['Enabled', 'Suspended']: raise MalformedXML() # Make sure the versions container exists req.container_name += VERSIONING_SUFFIX try: req.get_container_info(self.app) except NoSuchBucket: req.get_response(self.app, 'PUT', req.container_name, '') # Set up versioning if status == 'Enabled': req.headers['X-History-Location'] = req.container_name else: req.headers['X-Remove-History-Location'] = 'true' # Set the container back to what it originally was req.container_name = req.container_name[:-len(VERSIONING_SUFFIX)] resp = req.get_response(self.app, 'POST') return convert_response(req, resp, 204, HTTPOk)
def GET(self, req): """ Handles GET Bucket versioning. """ log_s3api_command(req, 'get-bucket-versioning') info = req.get_container_info(self.app) status = None versions_container = info.get('sysmeta', {}).get('versions-location') if versions_container: status = 'Enabled' else: versions_container = ''.join( [req.container_name, VERSIONING_SUFFIX]) try: req.get_response(self.app, 'HEAD', container=versions_container) status = 'Suspended' except NoSuchBucket: pass # Just report there is no versioning configured here. elem = Element('VersioningConfiguration') if status: SubElement(elem, 'Status').text = status body = tostring(elem) return HTTPOk(body=body, content_type="text/plain")
def PUT(self, req): """ Handle PUT Bucket request """ log_s3api_command(req, 'create-bucket') xml = req.xml(MAX_PUT_BUCKET_BODY_SIZE) if xml: # check location try: elem = fromstring(xml, 'CreateBucketConfiguration') location = elem.find('./LocationConstraint').text except (XMLSyntaxError, DocumentInvalid): raise MalformedXML() except Exception as e: LOGGER.error(e) raise if location != CONF.location: # Swift3 cannot support multiple regions currently. raise InvalidLocationConstraint() resp = req.get_response(self.app) # create bucket+segments to avoid breaking # functional tests for swift3 try: cnt = req.container_name + MULTIUPLOAD_SUFFIX req.get_response(self.app, 'PUT', cnt, '') except BucketAlreadyExists: pass resp.status = HTTP_OK resp.location = '/' + req.container_name return resp
def POST(self, req): """ Handles Initiate Multipart Upload. """ log_s3api_command(req, 'create-multipart-upload') # Create a unique S3 upload id from UUID to avoid duplicates. upload_id = unique_id() container = req.container_name + MULTIUPLOAD_SUFFIX obj = '%s/%s' % (req.object_name, upload_id) if HTTP_HEADER_TAGGING_KEY in req.headers: tagging = convert_urlquery_to_xml( req.headers.get(HTTP_HEADER_TAGGING_KEY)) req.headers[OBJECT_TAGGING_HEADER] = tagging req.headers.pop('Etag', None) req.headers.pop('Content-Md5', None) req.environ['oio.ephemeral_object'] = True req.get_response(self.app, 'PUT', container, obj, body='') result_elem = Element('InitiateMultipartUploadResult') SubElement(result_elem, 'Bucket').text = req.container_name SubElement(result_elem, 'Key').text = req.object_name SubElement(result_elem, 'UploadId').text = upload_id body = tostring(result_elem) return HTTPOk(body=body, content_type='application/xml')
def PUT(self, req): """ Handle PUT Object and PUT Object (Copy) request """ method = 'put-object' # set X-Timestamp by swift3 to use at copy resp body req_timestamp = S3Timestamp.now() req.headers['X-Timestamp'] = req_timestamp.internal if all(h in req.headers for h in ('X-Amz-Copy-Source', 'X-Amz-Copy-Source-Range')): raise InvalidArgument('x-amz-copy-source-range', req.headers['X-Amz-Copy-Source-Range'], 'Illegal copy header') if HTTP_HEADER_TAGGING_KEY in req.headers: tagging = convert_urlquery_to_xml( req.headers.pop(HTTP_HEADER_TAGGING_KEY)) req.headers[OBJECT_TAGGING_HEADER] = tagging req.check_copy_source(self.app) resp = req.get_response(self.app) if 'X-Amz-Copy-Source' in req.headers: method = 'copy-object' resp.append_copy_resp_body(req.controller_name, req_timestamp.s3xmlformat) # delete object metadata from response for key in list(resp.headers.keys()): if key.startswith('x-amz-meta-'): del resp.headers[key] log_s3api_command(req, method) resp.status = HTTP_OK return resp
def DELETE(self, req): """ Handle DELETE Object request """ log_s3api_command(req, 'delete-object') try: query = req.gen_multipart_manifest_delete_query(self.app) req.headers['Content-Type'] = None # Ignore client content-type if req.params.get('versionId'): resp = self._delete_version(req, query) else: ctinfo = req.get_container_info(self.app) if ctinfo.get('sysmeta', {}).get('versions-mode') == 'history': # If the object is a manifest, and versioning is enabled, # we must not delete the parts! resp = req.get_response(self.app) else: resp = req.get_response(self.app, query=query) if query and resp.status_int == HTTP_OK: for chunk in resp.app_iter: pass # drain the bulk-deleter response resp.status = HTTP_NO_CONTENT resp.body = '' except NoSuchKey: # expect to raise NoSuchBucket when the bucket doesn't exist req.get_container_info(self.app) raise return resp
def GET(self, req): """ Handles GET Bucket acl and GET Object acl. """ log_s3api_command(req, 'get-acl') resp = req.get_response(self.app, method='HEAD') return get_acl(req.user_id, resp.headers)
def DELETE(self, req): # pylint: disable=invalid-name """ Handles DELETE Bucket CORS. """ log_s3api_command(req, 'delete-bucket-cors') req.headers[BUCKET_CORS_HEADER] = '' resp = req.get_response(self.app, 'POST') return convert_response(req, resp, 202, HTTPNoContent)
def DELETE(self, req): """ Handle DELETE Bucket request """ log_s3api_command(req, 'delete-bucket') if CONF.allow_multipart_uploads: self._delete_segments_bucket(req) resp = req.get_response(self.app) return resp
def HEAD(self, req): """ Handle HEAD Bucket (Get Metadata) request """ log_s3api_command(req, 'head-bucket') resp = req.get_response(self.app) return HTTPOk(headers=resp.headers)
def GET(self, req): # pylint: disable=invalid-name """ Handles GET Bucket CORS. """ log_s3api_command(req, 'get-bucket-cors') resp = req.get_response(self.app, method='HEAD') body = resp.sysmeta_headers.get(BUCKET_CORS_HEADER) if not body: raise NoSuchCORSConfiguration return HTTPOk(body=body, content_type='application/xml')
def HEAD(self, req): """ Handle HEAD Object request """ log_s3api_command(req, 'head-object') resp = self.GETorHEAD(req) if 'range' in req.headers: req_range = req.headers['range'] resp = self._gen_head_range_resp(req_range, resp) return resp
def GET(self, req): """ Handle GET Service request """ log_s3api_command(req, 'list-buckets') resp = req.get_response(self.app, query={'format': 'json'}) containers = json.loads(resp.body) containers = filter( lambda item: validate_bucket_name(item['name']), containers) # we don't keep the creation time of a bucket (s3cmd doesn't # work without that) so we use something bogus. elem = Element('ListAllMyBucketsResult') owner = SubElement(elem, 'Owner') SubElement(owner, 'ID').text = req.user_id SubElement(owner, 'DisplayName').text = req.user_id buckets = SubElement(elem, 'Buckets') for c in containers: if 'last_modified' in c: ts = last_modified_date_to_timestamp(c['last_modified']) creation_date = S3Timestamp(ts).s3xmlformat else: creation_date = '2009-02-03T16:45:09.000Z' if CONF.s3_acl and CONF.check_bucket_owner: try: cname = c['name'].encode('utf-8') c_resp = req.get_response(self.app, 'HEAD', cname) if 'X-Timestamp' in c_resp.sw_headers: creation_date = S3Timestamp( c_resp.sw_headers['X-Timestamp']).s3xmlformat except AccessDenied: continue except NoSuchBucket: continue bucket = SubElement(buckets, 'Bucket') SubElement(bucket, 'Name').text = c['name'] SubElement(bucket, 'CreationDate').text = creation_date body = tostring(elem) return HTTPOk(content_type='application/xml', body=body)
def DELETE(self, req): """ Handles Abort Multipart Upload. """ log_s3api_command(req, 'abort-multipart-upload') upload_id = req.params['uploadId'] _check_upload_info(req, self.app, upload_id) container = req.container_name + MULTIUPLOAD_SUFFIX # First check to see if this multi-part upload was already # completed. Look in the primary container, if the object exists, # then it was completed and we return an error here. obj = '%s/%s' % (req.object_name, upload_id) req.environ['oio.ephemeral_object'] = True try: req.get_response(self.app, container=container, obj=obj) finally: req.environ['oio.ephemeral_object'] = False # The completed object was not found so this # must be a multipart upload abort. # We must delete any uploaded segments for this UploadID and then # delete the object in the main container as well query = { 'format': 'json', 'prefix': '%s/%s/' % (req.object_name, upload_id), 'delimiter': '/', } # Force the master to be sure to fetch all uploaded parts req.environ.setdefault('oio.query', {}) req.environ['oio.query']['force_master'] = True resp = req.get_response(self.app, 'GET', container, '', query=query) # Iterate over the segment objects and delete them individually objects = json.loads(resp.body) for o in objects: container = req.container_name + MULTIUPLOAD_SUFFIX obj = o['name'].encode('utf-8') req.get_response(self.app, container=container, obj=obj) return HTTPNoContent()
def OPTIONS(self, req): log_s3api_command(req, 'options') origin = req.headers.get('Origin') if not origin: raise CORSOriginMissing() method = req.headers.get('Access-Control-Request-Method') if method not in CORS_ALLOWED_HTTP_METHOD: raise CORSInvalidAccessControlRequest(method=method) rule = get_cors(self.app, req, method, origin) # FIXME(mbo): we should raise also NoSuchCORSConfiguration if rule is None: raise CORSForbidden(method) resp = HTTPOk(body=None) del resp.headers['Content-Type'] return cors_fill_headers(req, resp, rule)
def PUT(self, req): # pylint: disable=invalid-name """ Handles PUT Bucket CORS. """ log_s3api_command(req, 'put-bucket-cors') xml = req.xml(MAX_CORS_BODY_SIZE) try: data = fromstring(xml, "CorsConfiguration") except (XMLSyntaxError, DocumentInvalid): raise MalformedXML() except Exception as exc: LOGGER.error(exc) raise # forbid wildcard for ExposeHeader check_cors_rule(data) req.headers[BUCKET_CORS_HEADER] = xml resp = req.get_response(self.app, 'POST') return convert_response(req, resp, 204, HTTPOk)
def DELETE(self, req): """ Handle DELETE Bucket request """ log_s3api_command(req, 'delete-bucket') try: resp = super(UniqueBucketController, self).DELETE(req) except NoSuchBucket: ct_owner = req.bucket_db.get_owner(req.container_name) if ct_owner == req.account: # The bucket used to be ours, but for some reason # it has not been released. req.bucket_db.release(req.container_name) raise if resp.is_success: # Container deletion succeeded, reset owner. req.bucket_db.release(req.container_name) return resp
def PUT(self, req): """ Handle PUT Bucket request """ log_s3api_command(req, 'create-bucket') # We are about to create a container, reserve its name. can_create = req.bucket_db.reserve(req.container_name, req.account) if not can_create: ct_owner = req.bucket_db.get_owner(req.container_name) if ct_owner == req.account: raise BucketAlreadyOwnedByYou(req.container_name) raise BucketAlreadyExists(req.container_name) try: resp = super(UniqueBucketController, self).PUT(req) except Exception: # Container creation failed, remove reservation. req.bucket_db.release(req.container_name) raise # Container creation succeeded, confirm reservation. req.bucket_db.set_owner(req.container_name, req.account) return resp
def PUT(self, req): """ Handles Upload Part and Upload Part Copy. """ if 'uploadId' not in req.params: raise InvalidArgument('ResourceType', 'partNumber', 'Unexpected query string parameter') part_number = self.parse_part_number(req) upload_id = req.params['uploadId'] _check_upload_info(req, self.app, upload_id) req.container_name += MULTIUPLOAD_SUFFIX req.object_name = '%s/%s/%d' % (req.object_name, upload_id, part_number) req_timestamp = S3Timestamp.now() req.headers['X-Timestamp'] = req_timestamp.internal source_resp = req.check_copy_source(self.app) method = 'upload-part' if 'X-Amz-Copy-Source' in req.headers and \ 'X-Amz-Copy-Source-Range' in req.headers: rng = req.headers['X-Amz-Copy-Source-Range'] method = 'upload-part-copy' header_valid = True try: rng_obj = Range(rng) if len(rng_obj.ranges) != 1: header_valid = False except ValueError: header_valid = False if not header_valid: err_msg = ('The x-amz-copy-source-range value must be of the ' 'form bytes=first-last where first and last are ' 'the zero-based offsets of the first and last ' 'bytes to copy') raise InvalidArgument('x-amz-source-range', rng, err_msg) source_size = int(source_resp.headers['Content-Length']) if not rng_obj.ranges_for_length(source_size): err_msg = ('Range specified is not valid for source object ' 'of size: %s' % source_size) raise InvalidArgument('x-amz-source-range', rng, err_msg) req.headers['Range'] = rng del req.headers['X-Amz-Copy-Source-Range'] if 'X-Amz-Copy-Source' in req.headers: # Clear some problematic headers that might be on the source req.headers.update({ sysmeta_header('object', 'etag'): '', 'X-Object-Sysmeta-Swift3-Etag': '', # for legacy data 'X-Object-Sysmeta-Slo-Etag': '', 'X-Object-Sysmeta-Slo-Size': '', 'X-Object-Sysmeta-Container-Update-Override-Etag': '', }) log_s3api_command(req, method) resp = req.get_response(self.app) if 'X-Amz-Copy-Source' in req.headers: resp.append_copy_resp_body(req.controller_name, req_timestamp.s3xmlformat) resp.status = 200 return resp
def POST(self, req): """ Handles Complete Multipart Upload. """ log_s3api_command(req, 'complete-multipart-upload') upload_id = req.params['uploadId'] resp = _get_upload_info(req, self.app, upload_id) headers = {} for key, val in resp.headers.iteritems(): _key = key.lower() if _key.startswith('x-amz-meta-'): headers['x-object-meta-' + _key[11:]] = val elif _key == 'content-type': headers['Content-Type'] = val for key, val in resp.sysmeta_headers.items(): _key = key.lower() if _key == OBJECT_TAGGING_HEADER.lower(): headers[key] = val # Query for the objects in the segments area to make sure it completed query = { 'format': 'json', 'prefix': '%s/%s/' % (req.object_name, upload_id), 'delimiter': '/' } # Force the master to be sure to fetch all uploaded parts req.environ.setdefault('oio.query', {}) req.environ['oio.query']['force_master'] = True container = req.container_name + MULTIUPLOAD_SUFFIX resp = req.get_response(self.app, 'GET', container, '', query=query) objinfo = json.loads(resp.body) # pylint: disable-msg=no-member objinfo.sort(key=lambda o: int(o['name'].split('/')[-1])) objtable = dict((o['name'].encode('utf-8'), { 'path': '/'.join(['', container, o['name']]), 'etag': o['hash'], 'size_bytes': o['bytes'] }) for o in objinfo) s3_etag_hasher = md5() manifest = [] previous_number = 0 try: xml = req.xml(MAX_COMPLETE_UPLOAD_BODY_SIZE) if not xml: raise InvalidRequest(msg='You must specify at least one part') if 'content-md5' in req.headers: # If an MD5 was provided, we need to verify it. # Note that S3Request already took care of translating to ETag if req.headers['etag'] != md5(xml).hexdigest(): raise BadDigest(content_md5=req.headers['content-md5']) # We're only interested in the body here, in the # multipart-upload controller -- *don't* let it get # plumbed down to the object-server del req.headers['etag'] complete_elem = fromstring(xml, 'CompleteMultipartUpload') for part_elem in complete_elem.iterchildren('Part'): part_number = int(part_elem.find('./PartNumber').text) if part_number <= previous_number: raise InvalidPartOrder(upload_id=upload_id) previous_number = part_number etag = part_elem.find('./ETag').text if len(etag) >= 2 and etag[0] == '"' and etag[-1] == '"': # strip double quotes etag = etag[1:-1] info = objtable.get("%s/%s/%s" % (req.object_name, upload_id, part_number)) if info is None or info['etag'] != etag: raise InvalidPart(upload_id=upload_id, part_number=part_number) s3_etag_hasher.update(binascii.a2b_hex(etag)) info['size_bytes'] = int(info['size_bytes']) manifest.append(info) except (XMLSyntaxError, DocumentInvalid): raise MalformedXML() except ErrorResponse: raise except Exception as e: LOGGER.error(e) raise s3_etag = '%s-%d' % (s3_etag_hasher.hexdigest(), len(manifest)) headers[sysmeta_header('object', 'etag')] = s3_etag # Leave base header value blank; SLO will populate c_etag = '; s3_etag=%s' % s3_etag headers['X-Object-Sysmeta-Container-Update-Override-Etag'] = c_etag # Following swift commit 7f636a5, zero-byte segments aren't allowed, # even as the final segment empty_seg = None if manifest[-1]['size_bytes'] == 0: empty_seg = manifest.pop() # We'll check the sizes of all except the last segment below, but # since we just popped off a zero-byte segment, we should check # that last segment, too. if manifest and manifest[-1]['size_bytes'] < CONF.min_segment_size: raise EntityTooSmall() # Check the size of each segment except the last and make sure they are # all more than the minimum upload chunk size for info in manifest[:-1]: if info['size_bytes'] < CONF.min_segment_size: raise EntityTooSmall() try: # TODO: add support for versioning if manifest: resp = req.get_response(self.app, 'PUT', body=json.dumps(manifest), query={'multipart-manifest': 'put'}, headers=headers) else: # the upload must have consisted of a single zero-length part # just write it directly resp = req.get_response(self.app, 'PUT', body='', headers=headers) except ErrorResponse as e: msg = str(e._msg) expected_msg = 'too small; each segment must be at least 1 byte' if expected_msg in msg: # FIXME: AWS S3 allows a smaller object than 5 MB if there is # only one part. Use a COPY request to copy the part object # from the segments container instead. raise EntityTooSmall(msg) else: raise if empty_seg: # clean up the zero-byte segment _, empty_seg_cont, empty_seg_name = empty_seg['path'].split('/', 2) req.get_response(self.app, 'DELETE', container=empty_seg_cont, obj=empty_seg_name) # clean up the multipart-upload record obj = '%s/%s' % (req.object_name, upload_id) req.environ['oio.ephemeral_object'] = True req.get_response(self.app, 'DELETE', container, obj) result_elem = Element('CompleteMultipartUploadResult') # NOTE: boto with sig v4 appends port to HTTP_HOST value at the # request header when the port is non default value and it makes # req.host_url like as http://localhost:8080:8080/path # that obviously invalid. Probably it should be resolved at # swift.common.swob though, tentatively we are parsing and # reconstructing the correct host_url info here. # in detail, https://github.com/boto/boto/pull/3513 parsed_url = urlparse(req.host_url) host_url = '%s://%s' % (parsed_url.scheme, parsed_url.hostname) if parsed_url.port: host_url += ':%s' % parsed_url.port SubElement(result_elem, 'Location').text = host_url + req.path SubElement(result_elem, 'Bucket').text = req.container_name SubElement(result_elem, 'Key').text = req.object_name SubElement(result_elem, 'ETag').text = '"%s"' % s3_etag del resp.headers['ETag'] resp.body = tostring(result_elem) resp.status = 200 resp.content_type = "application/xml" return resp
def GET(self, req): """ Handles List Parts. """ log_s3api_command(req, 'list-parts') def filter_part_num_marker(o): try: num = int(os.path.basename(o['name'])) return num > part_num_marker except ValueError: return False encoding_type = req.params.get('encoding-type') if encoding_type is not None and encoding_type != 'url': err_msg = 'Invalid Encoding Method specified in Request' raise InvalidArgument('encoding-type', encoding_type, err_msg) upload_id = req.params['uploadId'] _check_upload_info(req, self.app, upload_id) maxparts = req.get_validated_param('max-parts', DEFAULT_MAX_PARTS_LISTING, CONF.max_parts_listing) part_num_marker = req.get_validated_param('part-number-marker', 0) query = { 'format': 'json', 'limit': maxparts + 1, 'prefix': '%s/%s/' % (req.object_name, upload_id), 'delimiter': '/' } container = req.container_name + MULTIUPLOAD_SUFFIX resp = req.get_response(self.app, container=container, obj='', query=query) objects = json.loads(resp.body) last_part = 0 # If the caller requested a list starting at a specific part number, # construct a sub-set of the object list. objList = filter(filter_part_num_marker, objects) # pylint: disable-msg=E1103 objList.sort(key=lambda o: int(o['name'].split('/')[-1])) if len(objList) > maxparts: objList = objList[:maxparts] truncated = True else: truncated = False # TODO: We have to retrieve object list again when truncated is True # and some objects filtered by invalid name because there could be no # enough objects for limit defined by maxparts. if objList: o = objList[-1] last_part = os.path.basename(o['name']) result_elem = Element('ListPartsResult') if encoding_type is not None: result_elem.encoding_type = encoding_type SubElement(result_elem, 'Bucket').text = req.container_name SubElement(result_elem, 'Key').text = req.object_name SubElement(result_elem, 'UploadId').text = upload_id initiator_elem = SubElement(result_elem, 'Initiator') SubElement(initiator_elem, 'ID').text = req.user_id SubElement(initiator_elem, 'DisplayName').text = req.user_id owner_elem = SubElement(result_elem, 'Owner') SubElement(owner_elem, 'ID').text = req.user_id SubElement(owner_elem, 'DisplayName').text = req.user_id SubElement(result_elem, 'StorageClass').text = 'STANDARD' SubElement(result_elem, 'PartNumberMarker').text = str(part_num_marker) SubElement(result_elem, 'NextPartNumberMarker').text = str(last_part) SubElement(result_elem, 'MaxParts').text = str(maxparts) if 'encoding-type' in req.params: SubElement(result_elem, 'EncodingType').text = \ req.params['encoding-type'] SubElement(result_elem, 'IsTruncated').text = \ 'true' if truncated else 'false' for i in objList: part_elem = SubElement(result_elem, 'Part') SubElement(part_elem, 'PartNumber').text = i['name'].split('/')[-1] SubElement(part_elem, 'LastModified').text = \ i['last_modified'][:-3] + 'Z' SubElement(part_elem, 'ETag').text = '"%s"' % i['hash'] SubElement(part_elem, 'Size').text = str(i['bytes']) body = tostring(result_elem, encoding_type=encoding_type) return HTTPOk(body=body, content_type='application/xml')
def GET(self, req): """ Handles List Multipart Uploads """ log_s3api_command(req, 'list-multipart-uploads') def separate_uploads(uploads, prefix, delimiter): """ separate_uploads will separate uploads into non_delimited_uploads (a subset of uploads) and common_prefixes according to the specified delimiter. non_delimited_uploads is a list of uploads which exclude the delimiter. common_prefixes is a set of prefixes prior to the specified delimiter. Note that the prefix in the common_prefixes includes the delimiter itself. i.e. if '/' delimiter specified and then the uploads is consists of ['foo', 'foo/bar'], this function will return (['foo'], ['foo/']). :param uploads: A list of uploads dictionary :param prefix: A string of prefix reserved on the upload path. (i.e. the delimiter must be searched behind the prefix) :param delimiter: A string of delimiter to split the path in each upload :return (non_delimited_uploads, common_prefixes) """ (prefix, delimiter) = \ utf8encode(prefix, delimiter) non_delimited_uploads = [] common_prefixes = set() for upload in uploads: key = upload['key'] end = key.find(delimiter, len(prefix)) if end >= 0: common_prefix = key[:end + len(delimiter)] common_prefixes.add(common_prefix) else: non_delimited_uploads.append(upload) return non_delimited_uploads, sorted(common_prefixes) encoding_type = req.params.get('encoding-type') if encoding_type is not None and encoding_type != 'url': err_msg = 'Invalid Encoding Method specified in Request' raise InvalidArgument('encoding-type', encoding_type, err_msg) keymarker = req.params.get('key-marker', '') uploadid = req.params.get('upload-id-marker', '') maxuploads = req.get_validated_param('max-uploads', DEFAULT_MAX_UPLOADS, DEFAULT_MAX_UPLOADS) query = { 'format': 'json', 'limit': maxuploads + 1, } if uploadid and keymarker: query.update({'marker': '%s/%s' % (keymarker, uploadid)}) elif keymarker: query.update({'marker': '%s/~' % (keymarker)}) if 'prefix' in req.params: query.update({'prefix': req.params['prefix']}) container = req.container_name + MULTIUPLOAD_SUFFIX try: req.environ['oio.list_mpu'] = True resp = req.get_response(self.app, container=container, query=query) objects = json.loads(resp.body) except NoSuchBucket: # Assume NoSuchBucket as no uploads objects = [] def object_to_upload(object_info): obj, upid = object_info['name'].rsplit('/', 1) obj_dict = { 'key': obj, 'upload_id': upid, 'last_modified': object_info['last_modified'] } return obj_dict # uploads is a list consists of dict, {key, upload_id, last_modified} # Note that pattern matcher will drop whole segments objects like as # object_name/upload_id/1. pattern = re.compile('/[0-9]+$') uploads = [ object_to_upload(obj) for obj in objects if pattern.search(obj.get('name', '')) is None ] prefixes = [] if 'delimiter' in req.params: prefix = req.params.get('prefix', '') delimiter = req.params['delimiter'] uploads, prefixes = \ separate_uploads(uploads, prefix, delimiter) if len(uploads) > maxuploads: uploads = uploads[:maxuploads] truncated = True else: truncated = False nextkeymarker = '' nextuploadmarker = '' if len(uploads) > 1: nextuploadmarker = uploads[-1]['upload_id'] nextkeymarker = uploads[-1]['key'] result_elem = Element('ListMultipartUploadsResult') if encoding_type is not None: result_elem.encoding_type = encoding_type SubElement(result_elem, 'Bucket').text = req.container_name SubElement(result_elem, 'KeyMarker').text = keymarker SubElement(result_elem, 'UploadIdMarker').text = uploadid SubElement(result_elem, 'NextKeyMarker').text = nextkeymarker SubElement(result_elem, 'NextUploadIdMarker').text = nextuploadmarker if 'delimiter' in req.params: SubElement(result_elem, 'Delimiter').text = \ req.params['delimiter'] if 'prefix' in req.params: SubElement(result_elem, 'Prefix').text = req.params['prefix'] SubElement(result_elem, 'MaxUploads').text = str(maxuploads) if encoding_type is not None: SubElement(result_elem, 'EncodingType').text = encoding_type SubElement(result_elem, 'IsTruncated').text = \ 'true' if truncated else 'false' # TODO: don't show uploads which are initiated before this bucket is # created. for u in uploads: upload_elem = SubElement(result_elem, 'Upload') SubElement(upload_elem, 'Key').text = u['key'] SubElement(upload_elem, 'UploadId').text = u['upload_id'] initiator_elem = SubElement(upload_elem, 'Initiator') SubElement(initiator_elem, 'ID').text = req.user_id SubElement(initiator_elem, 'DisplayName').text = req.user_id owner_elem = SubElement(upload_elem, 'Owner') SubElement(owner_elem, 'ID').text = req.user_id SubElement(owner_elem, 'DisplayName').text = req.user_id SubElement(upload_elem, 'StorageClass').text = 'STANDARD' SubElement(upload_elem, 'Initiated').text = \ u['last_modified'][:-3] + 'Z' for p in prefixes: elem = SubElement(result_elem, 'CommonPrefixes') SubElement(elem, 'Prefix').text = p body = tostring(result_elem, encoding_type=encoding_type) return HTTPOk(body=body, content_type='application/xml')
def HEAD(self, req): """ Handles Head Part (regular HEAD but with ?part-number=N). """ log_s3api_command(req, 'head-object-part') return self.GETorHEAD(req)
def GET(self, req): """ Handles Get Part (regular Get but with ?part-number=N). """ log_s3api_command(req, 'get-object-part') return self.GETorHEAD(req)
def GET(self, req): """ Handle GET Bucket (List Objects) request """ max_keys = req.get_validated_param('max-keys', CONF.max_bucket_listing) # TODO: Separate max_bucket_listing and default_bucket_listing tag_max_keys = max_keys max_keys = min(max_keys, CONF.max_bucket_listing) encoding_type = req.params.get('encoding-type') if encoding_type is not None and encoding_type != 'url': err_msg = 'Invalid Encoding Method specified in Request' raise InvalidArgument('encoding-type', encoding_type, err_msg) query = { 'format': 'json', 'limit': max_keys + 1, } if 'marker' in req.params: query.update({'marker': req.params['marker']}) if 'prefix' in req.params: query.update({'prefix': req.params['prefix']}) if 'delimiter' in req.params: query.update({'delimiter': req.params['delimiter']}) # GET Bucket (List Objects) Version 2 parameters is_v2 = int(req.params.get('list-type', '1')) == 2 fetch_owner = False if is_v2: log_s3api_command(req, 'list-objects-v2') if 'start-after' in req.params: query.update({'marker': req.params['start-after']}) # continuation-token overrides start-after if 'continuation-token' in req.params: decoded = b64decode(req.params['continuation-token']) query.update({'marker': decoded}) if 'fetch-owner' in req.params: fetch_owner = config_true_value(req.params['fetch-owner']) else: log_s3api_command(req, 'list-objects') resp = req.get_response(self.app, query=query) objects = json.loads(resp.body) if 'versions' in req.params: req.container_name += VERSIONING_SUFFIX query['reverse'] = 'true' try: resp = req.get_response(self.app, query=query) versioned_objects = json.loads(resp.body) prefixes = set() for o in versioned_objects: if 'name' in o: # The name looks like this: # '%03x%s/%s' % (len(name), name, version) o['name'], o['version_id'] = \ o['name'][3:].rsplit('/', 1) else: prefixes.add(o['subdir']) # suppress duplicated prefixes for o in list(objects): if 'subdir' in o and o['subdir'] in prefixes: objects.remove(o) objects.extend(versioned_objects) except NoSuchBucket: # the bucket may not be versioned pass req.container_name = req.container_name[:-len(VERSIONING_SUFFIX)] objects.sort(key=lambda o: o.get('name') or o.get('subdir')) for o in objects: if 'subdir' not in o and not o.get('version_id'): info = req.get_object_info( self.app, object_name=o['name'].encode('utf-8')) o['sysmeta_version_id'] = info.get('sysmeta', {}).get( 'version-id', 'null') if 'versions' in req.params: elem = Element('ListVersionsResult') else: elem = Element('ListBucketResult') if encoding_type is not None: elem.encoding_type = encoding_type SubElement(elem, 'Name').text = req.container_name SubElement(elem, 'Prefix').text = req.params.get('prefix') # Filter objects according to version-id-marker and key-marker v_marker = req.params.get('version-id-marker') k_marker = req.params.get('key-marker') k_marker_matched = not bool(k_marker) if 'versions' in req.params and (v_marker or k_marker): to_delete = [] for i, o in enumerate(objects): if 'subdir' not in o: version_id = o.get('version_id', o.get('sysmeta_version_id', 'null')) if not k_marker_matched and k_marker != o['name']: to_delete.append(i) if k_marker == o['name']: k_marker_matched = True if k_marker == o['name'] and v_marker: if v_marker == version_id: v_marker = None to_delete.append(i) for i in reversed(to_delete): objects.pop(i) # in order to judge that truncated is valid, check whether # max_keys + 1 th element exists in swift. is_truncated = max_keys > 0 and len(objects) > max_keys objects = objects[:max_keys] if not is_v2: if 'versions' in req.params: SubElement(elem, 'KeyMarker').text = req.params.get( 'key-marker') SubElement(elem, 'VersionIdMarker').text = req.params.get( 'version-id-marker') else: SubElement(elem, 'Marker').text = req.params.get('marker') if is_truncated and 'delimiter' in req.params: if 'name' in objects[-1]: SubElement(elem, 'NextMarker').text = \ objects[-1]['name'] if 'subdir' in objects[-1]: SubElement(elem, 'NextMarker').text = \ objects[-1]['subdir'] else: if is_truncated: if 'name' in objects[-1]: SubElement(elem, 'NextContinuationToken').text = \ b64encode(objects[-1]['name'].encode('utf8')) if 'subdir' in objects[-1]: SubElement(elem, 'NextContinuationToken').text = \ b64encode(objects[-1]['subdir'].encode('utf8')) if 'continuation-token' in req.params: SubElement(elem, 'ContinuationToken').text = \ req.params['continuation-token'] if 'start-after' in req.params: SubElement(elem, 'StartAfter').text = \ req.params['start-after'] SubElement(elem, 'KeyCount').text = str(len(objects)) SubElement(elem, 'MaxKeys').text = str(tag_max_keys) if 'delimiter' in req.params: SubElement(elem, 'Delimiter').text = req.params['delimiter'] if encoding_type is not None: SubElement(elem, 'EncodingType').text = encoding_type SubElement(elem, 'IsTruncated').text = \ 'true' if is_truncated else 'false' for o in objects: if 'subdir' not in o: if 'versions' in req.params: version_id = o.get('version_id', o.get('sysmeta_version_id', 'null')) if o.get('content_type') == DELETE_MARKER_CONTENT_TYPE: contents = SubElement(elem, 'DeleteMarker') else: contents = SubElement(elem, 'Version') SubElement(contents, 'Key').text = \ o['name'].encode('utf-8') SubElement(contents, 'VersionId').text = version_id SubElement(contents, 'IsLatest').text = str( 'version_id' not in o).lower() else: contents = SubElement(elem, 'Contents') SubElement(contents, 'Key').text = \ o['name'].encode('utf-8') SubElement(contents, 'LastModified').text = \ o['last_modified'][:-3] + 'Z' if contents.tag != 'DeleteMarker': if 's3_etag' in o: # New-enough MUs are already in the right format etag = o['s3_etag'] elif 'slo_etag' in o: # SLOs may be in something *close* to the MU format etag = '"%s-N"' % o['slo_etag'].strip('"') else: etag = o['hash'] if len(etag) < 2 or etag[::len(etag) - 1] != '""': # Normal objects just use the MD5 etag = '"%s"' % o['hash'] # This also catches sufficiently-old SLOs, # but we have no way to identify those # from container listings # Otherwise, somebody somewhere (proxyfs, maybe?) # made this look like an RFC-compliant ETag; # we don't need to quote-wrap. SubElement(contents, 'ETag').text = etag SubElement(contents, 'Size').text = str(o['bytes']) if fetch_owner or not is_v2: owner = SubElement(contents, 'Owner') SubElement(owner, 'ID').text = req.user_id SubElement(owner, 'DisplayName').text = req.user_id if contents.tag != 'DeleteMarker': SubElement(contents, 'StorageClass').text = 'STANDARD' for o in objects: if 'subdir' in o: common_prefixes = SubElement(elem, 'CommonPrefixes') SubElement(common_prefixes, 'Prefix').text = \ o['subdir'].encode('utf-8') body = tostring(elem, encoding_type=encoding_type) resp = HTTPOk(body=body, content_type='application/xml') origin = req.headers.get('Origin') if origin: rule = get_cors(self.app, req, "GET", origin) if rule: cors_fill_headers(req, resp, rule) return resp
def POST(self, req): """ Handles Delete Multiple Objects. """ log_s3api_command(req, 'delete-objects') def object_key_iter(elem): for obj in elem.iterchildren('Object'): key = obj.find('./Key').text if not key: raise UserKeyMustBeSpecified() version = obj.find('./VersionId') if version is not None: version = version.text yield key, version try: xml = req.xml(MAX_MULTI_DELETE_BODY_SIZE) if not xml: raise MissingRequestBodyError() req.check_md5(xml) elem = fromstring(xml, 'Delete') quiet = elem.find('./Quiet') self.quiet = quiet is not None and quiet.text.lower() == 'true' delete_list = list(object_key_iter(elem)) if len(delete_list) > CONF.max_multi_delete_objects: raise MalformedXML() except (XMLSyntaxError, DocumentInvalid): raise MalformedXML() except ErrorResponse: raise except Exception as e: LOGGER.error(e) raise elem = Element('DeleteResult') # check bucket existence try: req.get_response(self.app, 'HEAD') except AccessDenied as error: body = self._gen_error_body(error, elem, delete_list) return HTTPOk(body=body) for key, version in delete_list: if version is not None: # TODO: delete the specific version of the object raise S3NotImplemented() req.object_name = key try: query = req.gen_multipart_manifest_delete_query(self.app) req.get_response(self.app, method='DELETE', query=query) except ErrorResponse as e: error = SubElement(elem, 'Error') SubElement(error, 'Key').text = key SubElement(error, 'Code').text = e.__class__.__name__ SubElement(error, 'Message').text = e._msg continue if not self.quiet: deleted = SubElement(elem, 'Deleted') SubElement(deleted, 'Key').text = key body = tostring(elem) return HTTPOk(body=body)
def GET(self, req): """ Handle GET Object request """ log_s3api_command(req, 'get-object') return self.GETorHEAD(req)