def PUT(self, app): if not self.obj: # Initiate Multipart Uploads (put +segment container) resp = self._handle_acl(app, 'HEAD') req_acl = ACL.from_headers(self.req.headers, resp.bucket_acl.owner, Owner(self.user_id, self.user_id)) acl_headers = encode_acl('object', req_acl) self.req.headers[sysmeta_header('object', 'tmpacl')] = \ acl_headers[sysmeta_header('object', 'acl')]
def PUT(self, app): if not self.acl_checked: resp = self._handle_acl(app, 'HEAD', obj='') req_acl = ACL.from_headers(self.req.headers, resp.bucket_acl.owner, Owner(self.user_id, self.user_id)) acl_headers = encode_acl('object', req_acl) self.req.headers[sysmeta_header('object', 'tmpacl')] = \ acl_headers[sysmeta_header('object', 'acl')] self.acl_checked = True
def PUT(self, app): container = self.req.container_name + MULTIUPLOAD_SUFFIX obj = '%s/%s' % (self.obj, self.req.params['uploadId']) self.req.environ['oio.ephemeral_object'] = True try: resp = self.req._get_response(app, 'HEAD', container, obj) finally: self.req.environ['oio.ephemeral_object'] = False self.req.headers[sysmeta_header('object', 'acl')] = \ resp.sysmeta_headers.get(sysmeta_header('object', 'tmpacl'))
def test_fake_swift_sysmeta(self): swift = FakeSwift() orig_headers = HeaderKeyDict() orig_headers.update({sysmeta_header('container', 'acl'): 'test', 'x-container-meta-foo': 'bar'}) swift.register(self.method, self.path, MagicMock(), orig_headers, None) self._check_headers(swift, self.method, self.path, orig_headers) new_headers = orig_headers.copy() del new_headers[sysmeta_header('container', 'acl').title()] swift.register(self.method, self.path, MagicMock(), new_headers, None) self._check_headers(swift, self.method, self.path, orig_headers)
def __init__(self, *args, **kwargs): swob.Response.__init__(self, *args, **kwargs) sw_sysmeta_headers = swob.HeaderKeyDict() sw_headers = swob.HeaderKeyDict() headers = HeaderKeyDict() self.is_slo = False for key, val in self.headers.iteritems(): _key = key.lower() if _key.startswith(sysmeta_prefix('object')) or \ _key.startswith(sysmeta_prefix('container')): sw_sysmeta_headers[key] = val else: sw_headers[key] = val # Handle swift headers for key, val in sw_headers.iteritems(): _key = key.lower() if _key.startswith('x-object-meta-'): # Note that AWS ignores user-defined headers with '=' in the # header name. We translated underscores to '=5F' on the way # in, though. headers['x-amz-meta-' + _key[14:].replace('=5f', '_')] = val elif _key in ('content-length', 'content-type', 'content-range', 'content-encoding', 'content-disposition', 'content-language', 'etag', 'last-modified', 'x-robots-tag', 'cache-control', 'expires', 'x-amz-version-id'): headers[key] = val elif _key == 'x-object-sysmeta-version-id': headers['x-amz-version-id'] = val elif _key == 'x-static-large-object': # for delete slo self.is_slo = config_true_value(val) # Check whether we stored the AWS-style etag on upload override_etag = sw_sysmeta_headers.get(sysmeta_header( 'object', 'etag')) if override_etag not in (None, ''): # Multipart uploads in AWS have ETags like # <MD5(part_etag1 || ... || part_etagN)>-<number of parts> headers['etag'] = override_etag elif self.is_slo and 'etag' in headers: # Many AWS clients use the presence of a '-' to decide whether # to attempt client-side download validation, so even if we # didn't store the AWS-style header, tack on a '-N'. (Use 'N' # because we don't actually know how many parts there are.) headers['etag'] += '-N' self.headers = headers if self.etag: # add double quotes to the etag header self.etag = self.etag # Used for pure swift header handling at the request layer self.sw_headers = sw_headers self.sysmeta_headers = sw_sysmeta_headers
def GETorHEAD(self, req): had_match = False for match_header in ('if-match', 'if-none-match'): if match_header not in req.headers: continue had_match = True for value in list_from_csv(req.headers[match_header]): if value.startswith('"') and value.endswith('"'): value = value[1:-1] if value.endswith('-N'): # Deal with fake S3-like etags for SLOs uploaded via Swift req.headers[match_header] += ', ' + value[:-2] if had_match: # Update where to look update_etag_is_at_header(req, sysmeta_header('object', 'etag')) object_name = req.object_name version_id = req.params.get('versionId') if version_id and version_id != 'null': # get a specific version in the versioning container req.container_name += VERSIONING_SUFFIX req.object_name = versioned_object_name( req.object_name, req.params.pop('versionId')) cors_rule = None if req.headers.get('Origin'): cors_rule = get_cors(self.app, req, req.method, req.headers.get('Origin')) try: resp = req.get_response(self.app) except NoSuchKey: resp = None if version_id and version_id != 'null': # if the specific version is not in the versioning container, # it might be the current version req.container_name = req.container_name[ :-len(VERSIONING_SUFFIX)] req.object_name = object_name info = req.get_object_info(self.app, object_name=object_name) if info.get('sysmeta', {}).get('version-id') == version_id: resp = req.get_response(self.app) if resp is None: raise if req.method == 'HEAD': resp.app_iter = None if 'x-amz-meta-deleted' in resp.headers: raise NoSuchKey(object_name) for key in ('content-type', 'content-language', 'expires', 'cache-control', 'content-disposition', 'content-encoding'): if 'response-' + key in req.params: resp.headers[key] = req.params['response-' + key] if cors_rule is not None: cors_fill_headers(req, resp, cors_rule) return resp
def test_encode_acl_object(self): acl = ACLPrivate(Owner(id='test:tester', name='test:tester')) acp = encode_acl('object', acl) header_value = json.loads(acp[sysmeta_header('object', 'acl')]) self.assertTrue('Owner' in header_value) self.assertTrue('Grant' in header_value) self.assertEqual('test:tester', header_value['Owner']) self.assertEqual(len(header_value['Grant']), 1)
def test_object_multipart_upload_initiate_s3acl(self): req = Request.blank('/bucket/object?uploads', environ={'REQUEST_METHOD': 'POST'}, headers={ 'Authorization': 'AWS test:tester:hmac', 'x-amz-acl': 'public-read', 'x-amz-meta-foo': 'bar' }) status, headers, body = self.call_swift3(req) fromstring(body, 'InitiateMultipartUploadResult') self.assertEquals(status.split()[0], '200') _, _, req_headers = self.swift.calls_with_headers[-1] self.assertEquals(req_headers.get('X-Object-Meta-Foo'), 'bar') tmpacl_header = req_headers.get(sysmeta_header('object', 'tmpacl')) self.assertTrue(tmpacl_header) acl_header = encode_acl( 'object', ACLPublicRead(Owner('test:tester', 'test:tester'))) self.assertEquals(acl_header.get(sysmeta_header('object', 'acl')), tmpacl_header)
def POST(self, req): """ Handles Initiate Multipart Upload. """ # Create a unique S3 upload id from UUID to avoid duplicates. upload_id = unique_id() container = req.container_name + MULTIUPLOAD_SUFFIX content_type = req.headers.get('Content-Type') if content_type: req.headers[sysmeta_header('object', 'has-content-type')] = 'yes' req.headers[ sysmeta_header('object', 'content-type')] = content_type else: req.headers[sysmeta_header('object', 'has-content-type')] = 'no' req.headers['Content-Type'] = 'application/directory' try: req.get_response(self.app, 'PUT', container, '') except BucketAlreadyExists: pass obj = '%s/%s' % (req.object_name, upload_id) req.headers.pop('Etag', None) req.headers.pop('Content-Md5', None) req.get_response(self.app, 'PUT', container, obj, body='') result_elem = Element('InitiateMultipartUploadResult') SubElement(result_elem, 'Bucket').text = req.container_name SubElement(result_elem, 'Key').text = req.object_name SubElement(result_elem, 'UploadId').text = upload_id body = tostring(result_elem) return HTTPOk(body=body, content_type='application/xml')
def test_decode_acl_object(self): access_control_policy = \ {'Owner': 'test:tester', 'Grant': [{'Permission': 'FULL_CONTROL', 'Grantee': 'test:tester'}]} headers = {sysmeta_header('object', 'acl'): json.dumps(access_control_policy)} acl = decode_acl('object', headers) self.assertEqual(type(acl), ACL) self.assertEqual(acl.owner.id, 'test:tester') self.assertEqual(len(acl.grants), 1) self.assertEqual(str(acl.grants[0].grantee), 'test:tester') self.assertEqual(acl.grants[0].permission, 'FULL_CONTROL')
def test_object_multipart_upload_complete_s3acl(self): acl_headers = encode_acl( 'object', ACLPublicRead(Owner('test:tester', 'test:tester'))) headers = {} headers[sysmeta_header('object', 'tmpacl')] = \ acl_headers.get(sysmeta_header('object', 'acl')) headers['X-Object-Meta-Foo'] = 'bar' self.swift.register('HEAD', '/v1/AUTH_test/bucket+segments/object/X', swob.HTTPOk, headers, None) req = Request.blank('/bucket/object?uploadId=X', environ={'REQUEST_METHOD': 'POST'}, headers={'Authorization': 'AWS test:tester:hmac'}, body=xml) status, headers, body = self.call_swift3(req) fromstring(body, 'CompleteMultipartUploadResult') self.assertEquals(status.split()[0], '200') _, _, headers = self.swift.calls_with_headers[-2] self.assertEquals(headers.get('X-Object-Meta-Foo'), 'bar') self.assertEquals( tostring( ACLPublicRead(Owner('test:tester', 'test:tester')).elem()), tostring(decode_acl('object', headers).elem()))
def decode_acl(resource, headers): """ Decode Swift metadata to an ACL instance. Given a resource type and HTTP headers, this method returns an ACL instance. """ value = '' key = sysmeta_header(resource, 'acl') if key in headers: value = headers[key] if value == '': # Fix me: In the case of value is empty or not dict instance, # I want an instance of Owner as None. # However, in the above process would occur error in reference # to an instance variable of Owner. return ACL(Owner(None, None), []) try: encode_value = json.loads(value) if not isinstance(encode_value, dict): return ACL(Owner(None, None), []) id = None name = None grants = [] if 'Owner' in encode_value: id = encode_value['Owner'] name = encode_value['Owner'] if 'Grant' in encode_value: for grant in encode_value['Grant']: grantee = None # pylint: disable-msg=E1101 for group in Group.__subclasses__(): if group.__name__ == grant['Grantee']: grantee = group() if not grantee: grantee = User(grant['Grantee']) permission = grant['Permission'] grants.append(Grant(grantee, permission)) return ACL(Owner(id, name), grants) except Exception as e: LOGGER.debug(e) pass raise InvalidSubresource((resource, 'acl', value))
def test_encode_acl_many_grant(self): headers = {} users = [] for i in range(0, 99): users.append('id=test:tester%s' % str(i)) users = ','.join(users) headers['x-amz-grant-read'] = users acl = ACL.from_headers(headers, Owner('test:tester', 'test:tester')) acp = encode_acl('container', acl) header_value = acp[sysmeta_header('container', 'acl')] header_value = json.loads(header_value) self.assertTrue('Owner' in header_value) self.assertTrue('Grant' in header_value) self.assertEqual('test:tester', header_value['Owner']) self.assertEqual(len(header_value['Grant']), 99)
def encode_acl(resource, acl): """ Encode an ACL instance to Swift metadata. Given a resource type and an ACL instance, this method returns HTTP headers, which can be used for Swift metadata. """ header_value = {"Owner": acl.owner.id} grants = [] for grant in acl.grants: grant = {"Permission": grant.permission, "Grantee": str(grant.grantee)} grants.append(grant) header_value.update({"Grant": grants}) headers = {} key = sysmeta_header(resource, 'acl') headers[key] = dumps(header_value, separators=(',', ':')) return headers
def encode_acl(resource, acl): """ Encode an ACL instance to Swift metadata. Given a resource type and an ACL instance, this method returns HTTP headers, which can be used for Swift metadata. """ header_value = {"Owner": acl.owner.id} grants = [] for grant in acl.grants: grant = {"Permission": grant.permission, "Grantee": str(grant.grantee)} grants.append(grant) header_value.update({"Grant": grants}) headers = {} key = sysmeta_header(resource, 'acl') headers[key] = json.dumps(header_value, separators=(',', ':')) return headers
def deleter(self): self.headers[sysmeta_header(resource, 'acl')] = ''
from six.moves.urllib.parse import parse_qs from swift.common.utils import close_if_possible, public from swift3.controllers.base import Controller, check_container_existence from swift3.etree import fromstring, tostring, DocumentInvalid, \ Element, SubElement, XMLSyntaxError from swift3.iam import check_iam_access from swift3.response import HTTPNoContent, HTTPOk, MalformedXML, \ NoSuchTagSet, InvalidArgument from swift3.utils import sysmeta_header HTTP_HEADER_TAGGING_KEY = "x-amz-tagging" SYSMETA_TAGGING_KEY = 'swift3-tagging' BUCKET_TAGGING_HEADER = sysmeta_header('bucket', 'tagging') OBJECT_TAGGING_HEADER = sysmeta_header('object', 'tagging') # Not a swift3 header, cannot use sysmeta_header() VERSION_ID_HEADER = 'X-Object-Sysmeta-Version-Id' # FIXME(FVE): compute better size estimation according to key/value limits # 10 tags with 128b key and 256b value should be 3840 + envelope MAX_TAGGING_BODY_SIZE = 8 * 1024 INVALID_TAGGING = 'An error occurred (InvalidArgument) when calling ' \ 'the PutObject operation: The header \'x-amz-tagging\' ' \ 'shall be encoded as UTF-8 then URLEncoded URL query ' \ 'parameters without tag name duplicates.'
def test_decode_acl_with_invalid_json(self): headers = {sysmeta_header('container', 'acl'): '['} self.assertRaises(InvalidSubresource, decode_acl, 'container', headers)
def test_decode_acl_empty_list(self): headers = {sysmeta_header('container', 'acl'): '[]'} acl = decode_acl('container', headers) self.assertEqual(type(acl), ACL) self.assertEqual(None, acl.owner.id) self.assertEqual(len(acl.grants), 0)
def PUT(self, req): """ Handles Upload Part and Upload Part Copy. """ if 'uploadId' not in req.params: raise InvalidArgument('ResourceType', 'partNumber', 'Unexpected query string parameter') part_number = self.parse_part_number(req) upload_id = req.params['uploadId'] _check_upload_info(req, self.app, upload_id) req.container_name += MULTIUPLOAD_SUFFIX req.object_name = '%s/%s/%d' % (req.object_name, upload_id, part_number) req_timestamp = S3Timestamp.now() req.headers['X-Timestamp'] = req_timestamp.internal source_resp = req.check_copy_source(self.app) method = 'upload-part' if 'X-Amz-Copy-Source' in req.headers and \ 'X-Amz-Copy-Source-Range' in req.headers: rng = req.headers['X-Amz-Copy-Source-Range'] method = 'upload-part-copy' header_valid = True try: rng_obj = Range(rng) if len(rng_obj.ranges) != 1: header_valid = False except ValueError: header_valid = False if not header_valid: err_msg = ('The x-amz-copy-source-range value must be of the ' 'form bytes=first-last where first and last are ' 'the zero-based offsets of the first and last ' 'bytes to copy') raise InvalidArgument('x-amz-source-range', rng, err_msg) source_size = int(source_resp.headers['Content-Length']) if not rng_obj.ranges_for_length(source_size): err_msg = ('Range specified is not valid for source object ' 'of size: %s' % source_size) raise InvalidArgument('x-amz-source-range', rng, err_msg) req.headers['Range'] = rng del req.headers['X-Amz-Copy-Source-Range'] if 'X-Amz-Copy-Source' in req.headers: # Clear some problematic headers that might be on the source req.headers.update({ sysmeta_header('object', 'etag'): '', 'X-Object-Sysmeta-Swift3-Etag': '', # for legacy data 'X-Object-Sysmeta-Slo-Etag': '', 'X-Object-Sysmeta-Slo-Size': '', 'X-Object-Sysmeta-Container-Update-Override-Etag': '', }) log_s3api_command(req, method) resp = req.get_response(self.app) if 'X-Amz-Copy-Source' in req.headers: resp.append_copy_resp_body(req.controller_name, req_timestamp.s3xmlformat) resp.status = 200 return resp
def GETorHEAD(self, req): """ Handled GET or HEAD request on a part of a multipart object. """ part_number = self.parse_part_number(req) had_match = False for match_header in ('if-match', 'if-none-match'): if match_header not in req.headers: continue had_match = True for value in list_from_csv(req.headers[match_header]): if value.startswith('"') and value.endswith('"'): value = value[1:-1] if value.endswith('-N'): # Deal with fake S3-like etags for SLOs uploaded via Swift req.headers[match_header] += ', ' + value[:-2] if had_match: # Update where to look update_etag_is_at_header(req, sysmeta_header('object', 'etag')) # Get the list of parts. Must be raw to get all response headers. slo_resp = req.get_response(self.app, 'GET', req.container_name, req.object_name, query={ 'multipart-manifest': 'get', 'format': 'raw' }) # Check if the object is really a SLO. If not, and user asked # for the first part, do a regular request. if 'X-Static-Large-Object' not in slo_resp.sw_headers: if part_number == 1: if slo_resp.is_success and req.method == 'HEAD': # Clear body slo_resp.body = '' return slo_resp else: close_if_possible(slo_resp.app_iter) raise InvalidRange() # Locate the part slo = json.loads(slo_resp.body) try: part = slo[part_number - 1] except IndexError: raise InvalidRange() # Redirect the request on the part _, req.container_name, req.object_name = part['path'].split('/', 2) # XXX enforce container_name and object_name to be <str> # or it will rise issues in swift3/requests when merging both req.container_name = req.container_name.encode('utf-8') req.object_name = req.object_name.encode('utf8') # The etag check was performed with the manifest if had_match: for match_header in ('if-match', 'if-none-match'): req.headers.pop(match_header, None) resp = req.get_response(self.app) # Replace status slo_resp.status = resp.status # Replace body slo_resp.app_iter = resp.app_iter # Update with the size of the part slo_resp.headers['Content-Length'] = \ resp.headers.get('Content-Length', 0) slo_resp.sw_headers['Content-Length'] = \ slo_resp.headers['Content-Length'] # Add the number of parts in this object slo_resp.headers['X-Amz-Mp-Parts-Count'] = len(slo) return slo_resp
def POST(self, req): """ Handles Complete Multipart Upload. """ log_s3api_command(req, 'complete-multipart-upload') upload_id = req.params['uploadId'] resp = _get_upload_info(req, self.app, upload_id) headers = {} for key, val in resp.headers.iteritems(): _key = key.lower() if _key.startswith('x-amz-meta-'): headers['x-object-meta-' + _key[11:]] = val elif _key == 'content-type': headers['Content-Type'] = val for key, val in resp.sysmeta_headers.items(): _key = key.lower() if _key == OBJECT_TAGGING_HEADER.lower(): headers[key] = val # Query for the objects in the segments area to make sure it completed query = { 'format': 'json', 'prefix': '%s/%s/' % (req.object_name, upload_id), 'delimiter': '/' } # Force the master to be sure to fetch all uploaded parts req.environ.setdefault('oio.query', {}) req.environ['oio.query']['force_master'] = True container = req.container_name + MULTIUPLOAD_SUFFIX resp = req.get_response(self.app, 'GET', container, '', query=query) objinfo = json.loads(resp.body) # pylint: disable-msg=no-member objinfo.sort(key=lambda o: int(o['name'].split('/')[-1])) objtable = dict((o['name'].encode('utf-8'), { 'path': '/'.join(['', container, o['name']]), 'etag': o['hash'], 'size_bytes': o['bytes'] }) for o in objinfo) s3_etag_hasher = md5() manifest = [] previous_number = 0 try: xml = req.xml(MAX_COMPLETE_UPLOAD_BODY_SIZE) if not xml: raise InvalidRequest(msg='You must specify at least one part') if 'content-md5' in req.headers: # If an MD5 was provided, we need to verify it. # Note that S3Request already took care of translating to ETag if req.headers['etag'] != md5(xml).hexdigest(): raise BadDigest(content_md5=req.headers['content-md5']) # We're only interested in the body here, in the # multipart-upload controller -- *don't* let it get # plumbed down to the object-server del req.headers['etag'] complete_elem = fromstring(xml, 'CompleteMultipartUpload') for part_elem in complete_elem.iterchildren('Part'): part_number = int(part_elem.find('./PartNumber').text) if part_number <= previous_number: raise InvalidPartOrder(upload_id=upload_id) previous_number = part_number etag = part_elem.find('./ETag').text if len(etag) >= 2 and etag[0] == '"' and etag[-1] == '"': # strip double quotes etag = etag[1:-1] info = objtable.get("%s/%s/%s" % (req.object_name, upload_id, part_number)) if info is None or info['etag'] != etag: raise InvalidPart(upload_id=upload_id, part_number=part_number) s3_etag_hasher.update(binascii.a2b_hex(etag)) info['size_bytes'] = int(info['size_bytes']) manifest.append(info) except (XMLSyntaxError, DocumentInvalid): raise MalformedXML() except ErrorResponse: raise except Exception as e: LOGGER.error(e) raise s3_etag = '%s-%d' % (s3_etag_hasher.hexdigest(), len(manifest)) headers[sysmeta_header('object', 'etag')] = s3_etag # Leave base header value blank; SLO will populate c_etag = '; s3_etag=%s' % s3_etag headers['X-Object-Sysmeta-Container-Update-Override-Etag'] = c_etag # Following swift commit 7f636a5, zero-byte segments aren't allowed, # even as the final segment empty_seg = None if manifest[-1]['size_bytes'] == 0: empty_seg = manifest.pop() # We'll check the sizes of all except the last segment below, but # since we just popped off a zero-byte segment, we should check # that last segment, too. if manifest and manifest[-1]['size_bytes'] < CONF.min_segment_size: raise EntityTooSmall() # Check the size of each segment except the last and make sure they are # all more than the minimum upload chunk size for info in manifest[:-1]: if info['size_bytes'] < CONF.min_segment_size: raise EntityTooSmall() try: # TODO: add support for versioning if manifest: resp = req.get_response(self.app, 'PUT', body=json.dumps(manifest), query={'multipart-manifest': 'put'}, headers=headers) else: # the upload must have consisted of a single zero-length part # just write it directly resp = req.get_response(self.app, 'PUT', body='', headers=headers) except ErrorResponse as e: msg = str(e._msg) expected_msg = 'too small; each segment must be at least 1 byte' if expected_msg in msg: # FIXME: AWS S3 allows a smaller object than 5 MB if there is # only one part. Use a COPY request to copy the part object # from the segments container instead. raise EntityTooSmall(msg) else: raise if empty_seg: # clean up the zero-byte segment _, empty_seg_cont, empty_seg_name = empty_seg['path'].split('/', 2) req.get_response(self.app, 'DELETE', container=empty_seg_cont, obj=empty_seg_name) # clean up the multipart-upload record obj = '%s/%s' % (req.object_name, upload_id) req.environ['oio.ephemeral_object'] = True req.get_response(self.app, 'DELETE', container, obj) result_elem = Element('CompleteMultipartUploadResult') # NOTE: boto with sig v4 appends port to HTTP_HOST value at the # request header when the port is non default value and it makes # req.host_url like as http://localhost:8080:8080/path # that obviously invalid. Probably it should be resolved at # swift.common.swob though, tentatively we are parsing and # reconstructing the correct host_url info here. # in detail, https://github.com/boto/boto/pull/3513 parsed_url = urlparse(req.host_url) host_url = '%s://%s' % (parsed_url.scheme, parsed_url.hostname) if parsed_url.port: host_url += ':%s' % parsed_url.port SubElement(result_elem, 'Location').text = host_url + req.path SubElement(result_elem, 'Bucket').text = req.container_name SubElement(result_elem, 'Key').text = req.object_name SubElement(result_elem, 'ETag').text = '"%s"' % s3_etag del resp.headers['ETag'] resp.body = tostring(result_elem) resp.status = 200 resp.content_type = "application/xml" return resp
def PUT(self, app): container = self.req.container_name + MULTIUPLOAD_SUFFIX obj = '%s/%s' % (self.obj, self.req.params['uploadId']) resp = self.req._get_response(app, 'HEAD', container, obj) self.req.headers[sysmeta_header('object', 'acl')] = \ resp.sysmeta_headers.get(sysmeta_header('object', 'tmpacl'))
from swift.common.utils import public from swift3.controllers.base import Controller, bucket_operation, \ check_container_existence from swift3.etree import fromstring, DocumentInvalid, XMLSyntaxError from swift3.iam import check_iam_access from swift3.response import HTTPOk, HTTPNoContent, MalformedXML, \ NoSuchCORSConfiguration, CORSInvalidRequest from swift3.utils import LOGGER, sysmeta_header, log_s3api_command, \ convert_response MAX_CORS_BODY_SIZE = 10240 BUCKET_CORS_HEADER = sysmeta_header('bucket', 'cors') CORS_ALLOWED_HTTP_METHOD = ('GET', 'POST', 'PUT', 'HEAD', 'DELETE') def match(pattern, value): '''helper function for wildcard''' if '*' not in pattern: return pattern == value # protect dot as we keep them as is pattern = pattern.replace('.', '\\.') pattern = '^' + pattern.replace('*', '.*') + '$' return re.match(pattern, value) is not None def get_cors(app, req, method, origin):
# Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or # implied. # See the License for the specific language governing permissions and # limitations under the License. from swift.common.utils import public from swift3.controllers.base import Controller, bucket_operation from swift3.etree import fromstring, DocumentInvalid, XMLSyntaxError from swift3.response import HTTPOk, \ NoSuchLifecycleConfiguration, MalformedXML from swift3.utils import sysmeta_header LIFECYCLE_HEADER = sysmeta_header('container', 'lifecycle') MAX_LIFECYCLE_BODY_SIZE = 64 * 1024 # Arbitrary class LifecycleController(Controller): """ Handles the following APIs: - GET Bucket lifecycle - PUT Bucket lifecycle - DELETE Bucket lifecycle """ @public @bucket_operation(err_resp=NoSuchLifecycleConfiguration) def GET(self, req):
def test_decode_acl_empty_list(self): headers = {sysmeta_header('container', 'acl'): '[]'} acl = decode_acl('container', headers) self.assertEqual(type(acl), ACL) self.assertIsNone(acl.owner.id) self.assertEqual(len(acl.grants), 0)
def POST(self, req): """ Handles Complete Multipart Upload. """ upload_id = req.params['uploadId'] resp = _get_upload_info(req, self.app, upload_id) headers = {} for key, val in resp.headers.iteritems(): _key = key.lower() if _key.startswith('x-amz-meta-'): headers['x-object-meta-' + _key[11:]] = val hct_header = sysmeta_header('object', 'has-content-type') if resp.sysmeta_headers.get(hct_header) == 'yes': content_type = resp.sysmeta_headers.get( sysmeta_header('object', 'content-type')) elif hct_header in resp.sysmeta_headers: # has-content-type is present but false, so no content type was # set on initial upload. In that case, we won't set one on our # PUT request. Swift will end up guessing one based on the # object name. content_type = None else: content_type = resp.headers.get('Content-Type') if content_type: headers['Content-Type'] = content_type # Query for the objects in the segments area to make sure it completed query = { 'format': 'json', 'prefix': '%s/%s/' % (req.object_name, upload_id), 'delimiter': '/' } container = req.container_name + MULTIUPLOAD_SUFFIX resp = req.get_response(self.app, 'GET', container, '', query=query) objinfo = json.loads(resp.body) objtable = dict((o['name'], {'path': '/'.join(['', container, o['name']]), 'etag': o['hash'], 'size_bytes': o['bytes']}) for o in objinfo) manifest = [] previous_number = 0 try: xml = req.xml(MAX_COMPLETE_UPLOAD_BODY_SIZE) if not xml: raise InvalidRequest(msg='You must specify at least one part') complete_elem = fromstring(xml, 'CompleteMultipartUpload') for part_elem in complete_elem.iterchildren('Part'): part_number = int(part_elem.find('./PartNumber').text) if part_number <= previous_number: raise InvalidPartOrder(upload_id=upload_id) previous_number = part_number etag = part_elem.find('./ETag').text if len(etag) >= 2 and etag[0] == '"' and etag[-1] == '"': # strip double quotes etag = etag[1:-1] info = objtable.get("%s/%s/%s" % (req.object_name, upload_id, part_number)) if info is None or info['etag'] != etag: raise InvalidPart(upload_id=upload_id, part_number=part_number) info['size_bytes'] = int(info['size_bytes']) manifest.append(info) except (XMLSyntaxError, DocumentInvalid): raise MalformedXML() except ErrorResponse: raise except Exception as e: exc_type, exc_value, exc_traceback = sys.exc_info() LOGGER.error(e) raise exc_type, exc_value, exc_traceback # Following swift commit 7f636a5, zero-byte segments aren't allowed, # even as the final segment empty_seg = None if manifest[-1]['size_bytes'] == 0: empty_seg = manifest.pop() # We'll check the sizes of all except the last segment below, but # since we just popped off a zero-byte segment, we should check # that last segment, too. if manifest and manifest[-1]['size_bytes'] < CONF.min_segment_size: raise EntityTooSmall() # Check the size of each segment except the last and make sure they are # all more than the minimum upload chunk size for info in manifest[:-1]: if info['size_bytes'] < CONF.min_segment_size: raise EntityTooSmall() try: # TODO: add support for versioning if manifest: resp = req.get_response(self.app, 'PUT', body=json.dumps(manifest), query={'multipart-manifest': 'put'}, headers=headers) else: # the upload must have consisted of a single zero-length part # just write it directly resp = req.get_response(self.app, 'PUT', body='', headers=headers) except BadSwiftRequest as e: msg = str(e) expected_msg = 'too small; each segment must be at least 1 byte' if expected_msg in msg: # FIXME: AWS S3 allows a smaller object than 5 MB if there is # only one part. Use a COPY request to copy the part object # from the segments container instead. raise EntityTooSmall(msg) else: raise if empty_seg: # clean up the zero-byte segment _, empty_seg_cont, empty_seg_name = empty_seg['path'].split('/', 2) req.get_response(self.app, 'DELETE', container=empty_seg_cont, obj=empty_seg_name) # clean up the multipart-upload record obj = '%s/%s' % (req.object_name, upload_id) try: req.get_response(self.app, 'DELETE', container, obj) except NoSuchKey: pass # We know that this existed long enough for us to HEAD result_elem = Element('CompleteMultipartUploadResult') # NOTE: boto with sig v4 appends port to HTTP_HOST value at the # request header when the port is non default value and it makes # req.host_url like as http://localhost:8080:8080/path # that obviously invalid. Probably it should be resolved at # swift.common.swob though, tentatively we are parsing and # reconstructing the correct host_url info here. # in detail, https://github.com/boto/boto/pull/3513 parsed_url = urlparse(req.host_url) host_url = '%s://%s' % (parsed_url.scheme, parsed_url.hostname) if parsed_url.port: host_url += ':%s' % parsed_url.port SubElement(result_elem, 'Location').text = host_url + req.path SubElement(result_elem, 'Bucket').text = req.container_name SubElement(result_elem, 'Key').text = req.object_name SubElement(result_elem, 'ETag').text = resp.etag resp.body = tostring(result_elem) resp.status = 200 resp.content_type = "application/xml" return resp
def deleter(self): self.headers[sysmeta_header(resource, "acl")] = ""