def test_get_object_info_swift_source(self): app = FakeApp() req = Request.blank("/v1/a/c/o", environ={'swift.cache': FakeCache()}) get_object_info(req.environ, app, swift_source='LU') self.assertEqual([e['swift.source'] for e in app.captured_envs], ['LU'])
def object_request(self, req, api_version, account, container, obj, allow_versioned_writes): container_name = unquote(container) object_name = unquote(obj) orig_container = get_unversioned_container(container_name) if orig_container != container_name: orig_object, version = \ swift3_split_object_name_version(object_name) req.environ['oio.query'] = {'version': version} req.environ['PATH_INFO'] = '/%s/%s/%s/%s' % (api_version, account, quote(orig_container), quote(orig_object)) elif req.method == 'DELETE': ver_mode = req.headers.get('X-Backend-Versioning-Mode-Override', 'history') if ver_mode == 'stack': # Do not create a delete marker, delete the latest version obj_inf = get_object_info(req.environ, self.app, swift_source='VW') req.environ['oio.query'] = { 'version': obj_inf.get('sysmeta', {}).get('version-id') } resp = req.get_response(self.app) if req.method == 'HEAD': close_if_possible(resp.app_iter) return resp
def test_get_object_info_env(self): cached = {"status": 200, "length": 3333, "type": "application/json", "meta": {}} env_key = get_object_env_key("account", "cont", "obj") req = Request.blank("/v1/account/cont/obj", environ={env_key: cached, "swift.cache": FakeCache({})}) resp = get_object_info(req.environ, "xxx") self.assertEquals(resp["length"], 3333) self.assertEquals(resp["type"], "application/json")
def get_decryption_keys(self, req): """ Determine if a response should be decrypted, and if so then fetch keys. :param req: a Request object :returns: a dict of decryption keys """ if config_true_value(req.environ.get('swift.crypto.override')): self.logger.debug('No decryption is necessary because of override') return None info = get_object_info(req.environ, self.app, swift_source='DCRYPT') if 'crypto-etag' not in info['sysmeta']: # object is not cyphered return None try: return self.get_keys(req.environ) except HTTPException: # FIXME(FVE): check swift_source, accept if it is internal # FIXME(FVE): move that code to avoid printing an error if req.method in ('HEAD', 'GET'): try: return self.get_keys(req.environ, ['container']) except HTTPException: pass return None else: raise
def test_get_object_info_swift_source(self): req = Request.blank("/v1/a/c/o", environ={'swift.cache': FakeCache({})}) with patch('swift.proxy.controllers.base.' '_prepare_pre_auth_info_request', FakeRequest): resp = get_object_info(req.environ, 'app', swift_source='LU') self.assertEquals(resp['meta']['fakerequest-swift-source'], 'LU')
def __call__(self, env, start_response): """ If called with header X-Pid-Create and Method PUT become active and create a PID and store it with the object :param env: request environment :param start_response: function that we call when creating response :return: """ self.start_response = start_response request = Request(env) if request.method == 'PUT': if 'X-Pid-Create' in list(request.headers.keys()): url = '{}{}'.format(request.host_url, request.path_info) if 'X-Pid-Parent' in list(request.headers.keys()): parent = request.headers['X-Pid-Parent'] else: parent = None success, pid = create_pid(object_url=url, api_url=self.conf.get('api_url'), username=self.conf.get('username'), password=self.conf.get('password'), parent=parent) if success: self.logger.info('Created a PID for {}'.format(url)) request.headers['X-Object-Meta-PID'] = pid response = PersistentIdentifierResponse( pid=pid, add_checksum=self.add_checksum, username=self.conf.get('username'), password=self.conf.get('password'), start_response=start_response, logger=self.logger) return self.app(env, response.finish_response) else: self.logger.error('Unable to create a PID for {},' 'because of {}'.format(url, pid)) return Response( status=502, body='Could not contact PID API')(env, start_response) elif request.method in ['GET', 'HEAD']: # only modify response if we have a request for a object try: split_path(request.path_info, 4, 4, True) except ValueError: return self.app(env, start_response) object_metadata = get_object_info( env=request.environ, app=self.app, swift_source='PersistentIdentifierMiddleware')['meta'] if 'pid' in object_metadata.keys(): response = PersistentIdentifierResponse( pid='', add_checksum='', username='', password='', start_response=start_response, logger=self.logger) return self.app(env, response.finish_response_pidurl) return self.app(env, start_response)
def object_request(self, req, api_version, account, container, obj, allow_versioned_writes): container_name = unquote(container) object_name = unquote(obj) orig_container = get_unversioned_container(container_name) if orig_container != container_name: orig_object, version = \ swift3_split_object_name_version(object_name) req.environ['oio_query'] = {'version': version} req.environ['PATH_INFO'] = '/%s/%s/%s/%s' % (api_version, account, quote(orig_container), quote(orig_object)) elif req.method == 'DELETE': ver_mode = req.headers.get('X-Backend-Versioning-Mode-Override', 'history') if ver_mode == 'stack': # Do not create a delete marker, delete the latest version obj_inf = get_object_info(req.environ, self.app, swift_source='VW') req.environ['oio_query'] = { 'version': obj_inf.get('sysmeta', {}).get('version-id') } resp = req.get_response(self.app) if req.method == 'HEAD': close_if_possible(resp.app_iter) return resp
def PUT(self, req): """HTTP PUT request handler.""" container_info = self.container_info( self.account_name, self.container_name, req) req.acl = container_info['write_acl'] req.environ['swift_sync_key'] = container_info['sync_key'] # is request authorized if 'swift.authorize' in req.environ: aresp = req.environ['swift.authorize'](req) if aresp: return aresp old_slo_manifest = None # If versioning is disabled, we must check if the object exists. # If it's a SLO, we will have to delete the parts if the current # operation is a success. if (self.app.delete_slo_parts and not container_info['sysmeta'].get('versions-location', None)): try: dest_info = get_object_info(req.environ, self.app) if 'slo-size' in dest_info['sysmeta']: manifest_env = req.environ.copy() manifest_env['QUERY_STRING'] = 'multipart-manifest=get' manifest_req = make_subrequest(manifest_env, 'GET') manifest_resp = manifest_req.get_response(self.app) old_slo_manifest = json.loads(manifest_resp.body) except Exception as exc: self.app.logger.warn(('Failed to check existence of %s. If ' 'overwriting a SLO, old parts may ' 'remain. Error was: %s') % (req.path, exc)) self._update_content_type(req) self._update_x_timestamp(req) # check constraints on object name and request headers error_response = check_object_creation(req, self.object_name) or \ check_content_type(req) if error_response: return error_response if req.headers.get('Oio-Copy-From'): return self._link_object(req) data_source = req.environ['wsgi.input'] if req.content_length: data_source = ExpectedSizeReader(data_source, req.content_length) headers = self._prepare_headers(req) with closing_if_possible(data_source): resp = self._store_object(req, data_source, headers) if old_slo_manifest and resp.is_success: self.app.logger.debug( 'Previous object %s was a SLO, deleting parts', req.path) self._delete_slo_parts(req, old_slo_manifest) return resp
def test_get_object_info_swift_source(self): req = Request.blank("/v1/a/c/o", environ={'swift.cache': FakeCache({})}) with patch( 'swift.proxy.controllers.base.' '_prepare_pre_auth_info_request', FakeRequest): resp = get_object_info(req.environ, 'app', swift_source='LU') self.assertEquals(resp['meta']['fakerequest-swift-source'], 'LU')
def test_get_object_info_no_env(self): req = Request.blank("/v1/account/cont/obj", environ={'swift.cache': FakeCache({})}) with patch('swift.proxy.controllers.base.' '_prepare_pre_auth_info_request', FakeRequest): resp = get_object_info(req.environ, 'xxx') self.assertEquals(resp['length'], 5555) self.assertEquals(resp['type'], 'text/plain')
def __call__(self, request): if request.method not in ("POST", "PUT"): return self.app try: ver, account, container, obj = request.split_path( 2, 4, rest_with_last=True) except ValueError: return self.app if not container: # account request, so we pay attention to the quotas new_quota = request.headers.get('X-Account-Meta-Quota-Bytes') remove_quota = request.headers.get( 'X-Remove-Account-Meta-Quota-Bytes') else: # container or object request; even if the quota headers are set # in the request, they're meaningless new_quota = remove_quota = None if remove_quota: new_quota = 0 # X-Remove dominates if both are present if request.environ.get('reseller_request') is True: if new_quota and not new_quota.isdigit(): return HTTPBadRequest() return self.app # deny quota set for non-reseller if new_quota is not None: return HTTPForbidden() if obj and request.method == "POST" or not obj: return self.app copy_from = request.headers.get('X-Copy-From') content_length = (request.content_length or 0) if obj and copy_from: path = '/' + ver + '/' + account + '/' + copy_from.lstrip('/') object_info = get_object_info(request.environ, self.app, path) if not object_info or not object_info['length']: content_length = 0 else: content_length = int(object_info['length']) account_info = get_account_info(request.environ, self.app) if not account_info or not account_info['bytes']: return self.app new_size = int(account_info['bytes']) + content_length quota = int(account_info['meta'].get('quota-bytes', -1)) if 0 <= quota < new_size: return HTTPRequestEntityTooLarge() return self.app
def test_get_object_info_no_env(self): req = Request.blank("/v1/account/cont/obj", environ={'swift.cache': FakeCache({})}) with patch( 'swift.proxy.controllers.base.' '_prepare_pre_auth_info_request', FakeRequest): resp = get_object_info(req.environ, 'xxx') self.assertEquals(resp['length'], 5555) self.assertEquals(resp['type'], 'text/plain')
def test_get_object_info_no_env(self): app = FakeApp() req = Request.blank("/v1/account/cont/obj", environ={"swift.cache": FakeCache({})}) resp = get_object_info(req.environ, app) self.assertEqual(app.responses.stats["account"], 0) self.assertEqual(app.responses.stats["container"], 0) self.assertEqual(app.responses.stats["obj"], 1) self.assertEquals(resp["length"], 5555) self.assertEquals(resp["type"], "text/plain")
def __call__(self, request): if request.method not in ("POST", "PUT"): return self.app try: ver, account, container, obj = request.split_path(2, 4, rest_with_last=True) except ValueError: return self.app if not container: # account request, so we pay attention to the quotas new_quota = request.headers.get("X-Account-Meta-Quota-Bytes") remove_quota = request.headers.get("X-Remove-Account-Meta-Quota-Bytes") else: # container or object request; even if the quota headers are set # in the request, they're meaningless new_quota = remove_quota = None if remove_quota: new_quota = 0 # X-Remove dominates if both are present if request.environ.get("reseller_request") is True: if new_quota and not new_quota.isdigit(): return HTTPBadRequest() return self.app # deny quota set for non-reseller if new_quota is not None: return HTTPForbidden() if obj and request.method == "POST" or not obj: return self.app copy_from = request.headers.get("X-Copy-From") content_length = request.content_length or 0 if obj and copy_from: path = "/" + ver + "/" + account + "/" + copy_from.lstrip("/") object_info = get_object_info(request.environ, self.app, path) if not object_info or not object_info["length"]: content_length = 0 else: content_length = int(object_info["length"]) account_info = get_account_info(request.environ, self.app) if not account_info or not account_info["bytes"]: return self.app new_size = int(account_info["bytes"]) + content_length quota = int(account_info["meta"].get("quota-bytes", -1)) if 0 <= quota < new_size: return HTTPRequestEntityTooLarge() return self.app
def test_get_object_info_no_env(self): app = FakeApp() req = Request.blank("/v1/account/cont/obj", environ={'swift.cache': FakeCache({})}) resp = get_object_info(req.environ, app) self.assertEqual(app.responses.stats['account'], 0) self.assertEqual(app.responses.stats['container'], 0) self.assertEqual(app.responses.stats['obj'], 1) self.assertEqual(resp['length'], 5555) self.assertEqual(resp['type'], 'text/plain')
def test_get_object_info_env(self): cached = {'status': 200, 'length': 3333, 'type': 'application/json', 'meta': {}} env_key = get_object_env_key("account", "cont", "obj") req = Request.blank("/v1/account/cont/obj", environ={env_key: cached, 'swift.cache': FakeCache({})}) resp = get_object_info(req.environ, 'xxx') self.assertEquals(resp['length'], 3333) self.assertEquals(resp['type'], 'application/json')
def apply_storage_quota(self, req, service_plan, account_info, ver, account, container, obj): if not obj: quota = service_plan['containers'] # If "number of containers" = (quota + 1): deny PUT # We don't want to deny overwrite of the last container new_size = int(account_info['container_count']) if 0 <= quota < new_size: return bad_response( req, None, 'Over quota: containers') return None content_length = (req.content_length or 0) if req.method == 'COPY': copy_from = container + '/' + obj else: copy_from = req.headers.get('X-Copy-From') container_info = None if copy_from: copy_account = req.headers.get('X-Copy-From-Account', account) path = '/' + ver + '/' + copy_account + '/' + copy_from.lstrip('/') # We are copying from another account # Let's not leak the existence of the remote object # to the unauthorized user if copy_account != account: container_info = get_container_info(req.environ, self.app, swift_source='litequota') aresp = check_acl(req, container_info, 'read_acl') if aresp: return aresp object_info = get_object_info(req.environ, self.app, path) if not object_info or not object_info['length']: content_length = 0 else: content_length = int(object_info['length']) new_size = int(account_info['bytes']) + content_length quota = service_plan['bytes'] if 0 <= quota < new_size: if not container_info: container_info = get_container_info(req.environ, self.app, swift_source='litequota') return bad_response(req, container_info, 'Over quota: bytes') # If "number of objects" == (quota + 1): deny PUT # We don't want to deny overwrite of the last object new_size = int(account_info['total_object_count']) quota = service_plan['objects'] if 0 <= quota < new_size: if not container_info: container_info = get_container_info(req.environ, self.app, swift_source='litequota') return bad_response(req, container_info, 'Over quota: objects')
def __call__(self, req): try: (version, account, container, obj) = req.split_path(3, 4, True) except ValueError: return self.app # verify new quota headers are properly formatted if not obj and req.method in ("PUT", "POST"): val = req.headers.get("X-Container-Meta-Quota-Bytes") if val and not val.isdigit(): return HTTPBadRequest(body="Invalid bytes quota.") val = req.headers.get("X-Container-Meta-Quota-Count") if val and not val.isdigit(): return HTTPBadRequest(body="Invalid count quota.") # check user uploads against quotas elif obj and req.method == "PUT": container_info = get_container_info(req.environ, self.app, swift_source="CQ") if not container_info or not is_success(container_info["status"]): # this will hopefully 404 later return self.app if ( "quota-bytes" in container_info.get("meta", {}) and "bytes" in container_info and container_info["meta"]["quota-bytes"].isdigit() ): content_length = req.content_length or 0 copy_from = req.headers.get("X-Copy-From") if copy_from: path = "/%s/%s/%s" % (version, account, copy_from.lstrip("/")) object_info = get_object_info(req.environ, self.app, path) if not object_info or not object_info["length"]: content_length = 0 else: content_length = int(object_info["length"]) new_size = int(container_info["bytes"]) + content_length if int(container_info["meta"]["quota-bytes"]) < new_size: return self.bad_response(req, container_info) if ( "quota-count" in container_info.get("meta", {}) and "object_count" in container_info and container_info["meta"]["quota-count"].isdigit() ): new_count = int(container_info["object_count"]) + 1 if int(container_info["meta"]["quota-count"]) < new_count: return self.bad_response(req, container_info) return self.app
def __call__(self, req): try: (version, account, container, obj) = req.split_path(3, 4, True) except ValueError: return self.app # verify new quota headers are properly formatted if not obj and req.method in ('PUT', 'POST'): val = req.headers.get('X-Container-Meta-Quota-Bytes') if val and not val.isdigit(): return HTTPBadRequest(body='Invalid bytes quota.') val = req.headers.get('X-Container-Meta-Quota-Count') if val and not val.isdigit(): return HTTPBadRequest(body='Invalid count quota.') # check user uploads against quotas elif obj and req.method == 'PUT': container_info = get_container_info(req.environ, self.app, swift_source='CQ') if not container_info or not is_success(container_info['status']): # this will hopefully 404 later return self.app if 'quota-bytes' in container_info.get('meta', {}) and \ 'bytes' in container_info and \ container_info['meta']['quota-bytes'].isdigit(): content_length = (req.content_length or 0) copy_from = req.headers.get('X-Copy-From') if copy_from: path = '/%s/%s/%s' % (version, account, copy_from.lstrip('/')) object_info = get_object_info(req.environ, self.app, path) if not object_info or not object_info['length']: content_length = 0 else: content_length = int(object_info['length']) new_size = int(container_info['bytes']) + content_length if int(container_info['meta']['quota-bytes']) < new_size: return self.bad_response(req, container_info) if 'quota-count' in container_info.get('meta', {}) and \ 'object_count' in container_info and \ container_info['meta']['quota-count'].isdigit(): new_count = int(container_info['object_count']) + 1 if int(container_info['meta']['quota-count']) < new_count: return self.bad_response(req, container_info) return self.app
def __call__(self, req): try: (version, account, container, obj) = req.split_path(3, 4, True) except ValueError: return self.app # verify new quota headers are properly formatted if not obj and req.method in ('PUT', 'POST'): val = req.headers.get('X-Container-Meta-Quota-Bytes') if val and not val.isdigit(): return HTTPBadRequest(body='Invalid bytes quota.') val = req.headers.get('X-Container-Meta-Quota-Count') if val and not val.isdigit(): return HTTPBadRequest(body='Invalid count quota.') # check user uploads against quotas elif obj and req.method == 'PUT': container_info = get_container_info( req.environ, self.app, swift_source='CQ') if not container_info or not is_success(container_info['status']): # this will hopefully 404 later return self.app if 'quota-bytes' in container_info.get('meta', {}) and \ 'bytes' in container_info and \ container_info['meta']['quota-bytes'].isdigit(): content_length = (req.content_length or 0) if 'x-copy-from' in req.headers: src_cont, src_obj = check_copy_from_header(req) path = '/%s/%s/%s/%s' % (version, account, src_cont, src_obj) object_info = get_object_info(req.environ, self.app, path) if not object_info or not object_info['length']: content_length = 0 else: content_length = int(object_info['length']) new_size = int(container_info['bytes']) + content_length if int(container_info['meta']['quota-bytes']) < new_size: return self.bad_response(req, container_info) if 'quota-count' in container_info.get('meta', {}) and \ 'object_count' in container_info and \ container_info['meta']['quota-count'].isdigit(): new_count = int(container_info['object_count']) + 1 if int(container_info['meta']['quota-count']) < new_count: return self.bad_response(req, container_info) return self.app
def __call__(self, request): if request.method not in ("POST", "PUT"): return self.app try: ver, acc, cont, obj = request.split_path(2, 4, rest_with_last=True) except ValueError: return self.app new_quota = request.headers.get('X-Account-Meta-Quota-Bytes') remove_quota = request.headers.get('X-Remove-Account-Meta-Quota-Bytes') if remove_quota: new_quota = 0 # X-Remove dominates if both are present if request.environ.get('reseller_request') is True: if new_quota and not new_quota.isdigit(): return HTTPBadRequest() return self.app # deny quota set for non-reseller if new_quota is not None: return HTTPForbidden() copy_from = request.headers.get('X-Copy-From') content_length = (request.content_length or 0) if obj and copy_from: path = '/' + ver + '/' + acc + '/' + copy_from.lstrip('/') object_info = get_object_info(request.environ, self.app, path) if not object_info or not object_info['length']: content_length = 0 else: content_length = int(object_info['length']) account_info = get_account_info(request.environ, self.app) if not account_info or not account_info['bytes']: return self.app new_size = int(account_info['bytes']) + content_length quota = int(account_info['meta'].get('quota-bytes', -1)) if 0 <= quota < new_size: return HTTPRequestEntityTooLarge() return self.app
def subtract_quota_used(self, quota_info, env): # HEAD to swift resource to know the content length quota_limit = long(quota_info['quota_limit']) quota_used = long(quota_info['quota_used']) object_info = get_object_info(env, self.app, env['PATH_INFO']) if not object_info or not object_info['length']: content_to_delete = 0 else: content_to_delete = long(object_info['length']) quota_used_after_delete = quota_used - content_to_delete #send new quota to quota server self.app.logger.info('StackSync Quota: subtract_quota_used') response = self.rpc_server.XmlRpcQuotaHandler.updateAvailableQuota(quota_info['user'], str(quota_used_after_delete)) response = create_response(response, status_code=200) if not is_valid_status(response.status_int): self.app.logger.error("StackSync Quota: Error updating quota used") return response return self.app
def __call__(self, request): try: (version, account, container, objname) = split_path( request.path_info, 1, 4, True) except ValueError: return self.app if container and self.header_container_metadata: container_info = get_container_info(request.environ, self.app) for key in self.header_container_metadata: value = container_info.get('meta', {}).get(key) if value: keyname = 'X-CONTAINER-METADATA-%s' % key.upper() request.headers[keyname] = value if objname and self.header_object_metadata: object_info = get_object_info(request.environ, self.app) for key in self.header_object_metadata: value = object_info.get('meta', {}).get(key) if value: keyname = 'X-OBJECT-METADATA-%s' % key request.headers[keyname] = value return self.app
def _test_transient_sysmeta_replaced_by_PUT_or_POST(self, app): # check transient_sysmeta is replaced en-masse by a POST path = '/v1/a/c/o' env = {'REQUEST_METHOD': 'PUT'} hdrs = dict(self.original_transient_sysmeta_headers_1) hdrs.update(self.original_transient_sysmeta_headers_2) hdrs.update(self.original_meta_headers_1) req = Request.blank(path, environ=env, headers=hdrs, body='x') resp = req.get_response(app) self._assertStatus(resp, 201) req = Request.blank(path, environ={}) resp = req.get_response(app) self._assertStatus(resp, 200) self._assertInHeaders(resp, self.original_transient_sysmeta_headers_1) self._assertInHeaders(resp, self.original_transient_sysmeta_headers_2) self._assertInHeaders(resp, self.original_meta_headers_1) info = get_object_info(req.environ, app) self.assertEqual(2, len(info.get('transient_sysmeta', ()))) self.assertEqual({ 'testa': 'A', 'testb': 'B' }, info['transient_sysmeta']) # POST will replace all existing transient_sysmeta and usermeta values env = {'REQUEST_METHOD': 'POST'} hdrs = dict(self.changed_transient_sysmeta_headers) hdrs.update(self.new_transient_sysmeta_headers_1) req = Request.blank(path, environ=env, headers=hdrs) resp = req.get_response(app) self._assertStatus(resp, 202) req = Request.blank(path, environ={}) resp = req.get_response(app) self._assertStatus(resp, 200) self._assertInHeaders(resp, self.changed_transient_sysmeta_headers) self._assertInHeaders(resp, self.new_transient_sysmeta_headers_1) self._assertNotInHeaders(resp, self.original_meta_headers_1) self._assertNotInHeaders(resp, self.original_transient_sysmeta_headers_2) info = get_object_info(req.environ, app) self.assertEqual(2, len(info.get('transient_sysmeta', ()))) self.assertEqual({ 'testa': 'changed_A', 'testc': 'C' }, info['transient_sysmeta']) # subsequent PUT replaces all transient_sysmeta and usermeta values env = {'REQUEST_METHOD': 'PUT'} hdrs = dict(self.new_transient_sysmeta_headers_2) hdrs.update(self.original_meta_headers_2) req = Request.blank(path, environ=env, headers=hdrs, body='x') resp = req.get_response(app) self._assertStatus(resp, 201) req = Request.blank(path, environ={}) resp = req.get_response(app) self._assertStatus(resp, 200) self._assertInHeaders(resp, self.original_meta_headers_2) self._assertInHeaders(resp, self.new_transient_sysmeta_headers_2) # meta from previous POST should have gone away... self._assertNotInHeaders(resp, self.changed_transient_sysmeta_headers) self._assertNotInHeaders(resp, self.new_transient_sysmeta_headers_1) # sanity check that meta from first PUT did not re-appear... self._assertNotInHeaders(resp, self.original_meta_headers_1) self._assertNotInHeaders(resp, self.original_transient_sysmeta_headers_1) self._assertNotInHeaders(resp, self.original_transient_sysmeta_headers_2) info = get_object_info(req.environ, app) self.assertEqual(1, len(info.get('transient_sysmeta', ()))) self.assertEqual({'testd': 'D'}, info['transient_sysmeta'])
def __call__(self, env, start_response): req = Request(env) try: version, account, container, obj = req.split_path(4, 4, True) except ValueError: return self.app(env, start_response) container_info = get_container_info( req.environ, self.app, swift_source='ImageScalerMiddleware') # parse query string if req.query_string: qs = parse_qs(req.query_string) if 'size' in qs: req_size = qs['size'] else: self.logger.debug("image-scaler: No image scaling requested.") return self.app(env, start_response) else: # nothing for us to do, no scaling requested self.logger.debug("image-scaler: No image scaling requested.") return self.app(env, start_response) # check container whether scaling is allowed meta = container_info['meta'] if not meta.has_key('image-scaling') or \ meta.has_key('image-scaling') and \ not meta['image-scaling'].lower() in ['true', '1']: # nothing for us to do self.logger.debug("image-scaler: Image scaling not " "allowed. Nothing for us to do.") return self.app(env, start_response) # default allowed extensions allowed_exts = self.conf.get('formats', 'jpg;png;gif') allowed_exts = allowed_exts.lower() allowed_exts = allowed_exts.split(';') # check whether file has the allowed ending if meta.has_key('image-scaling-extensions'): allowed_exts = meta['image-scaling-extensions'].split(',') requested_ext = req.path.rsplit('.', 1)[-1] if not requested_ext.lower() in map(lambda x: x.lower(), allowed_exts): self.logger.info("image-scaler: extension %s not allowed" " for image scaling" % requested_ext) return self.app(env, start_response) # get maxsize from config, otherwise 20 MB max_size = self.conf.get('maxsize', '20971520') try: max_size = int(max_size) except ValueError: max_size = 20971520 self.logger.error("wrong format for max_size from configuration file, using 20 MB") if meta.has_key('image-scaling-max-size'): max_size = int(meta['image-scaling-max-size']) obj_info = get_object_info(req.environ, self.app, swift_source="ImageScalerMiddleware") if int(obj_info['length']) > max_size: self.logger.info("image-scaler: object too large") return self.app(env, start_response) response = ImageScalerResponse(start_response, req_size, self.logger) app_iter = self.app(env, response.scaler_start_response) if app_iter is not None: response.finish_response(app_iter) return response.write()
def __call__(self, req): try: (version, account, container, obj) = req.split_path(3, 4, True) except ValueError: return self.app # verify new quota headers are properly formatted if not obj and req.method in ('PUT', 'POST'): val = req.headers.get('X-Container-Meta-Quota-Bytes') if val and not val.isdigit(): return HTTPBadRequest(body='Invalid bytes quota.') val = req.headers.get('X-Container-Meta-Quota-Count') if val and not val.isdigit(): return HTTPBadRequest(body='Invalid count quota.') # check user uploads against quotas elif obj and req.method in ('PUT', 'COPY'): container_info = None if req.method == 'PUT': container_info = get_container_info( req.environ, self.app, swift_source='CQ') if req.method == 'COPY' and 'Destination' in req.headers: dest_account = account if 'Destination-Account' in req.headers: dest_account = req.headers.get('Destination-Account') dest_account = check_account_format(req, dest_account) dest_container, dest_object = check_destination_header(req) path_info = req.environ['PATH_INFO'] req.environ['PATH_INFO'] = "/%s/%s/%s/%s" % ( version, dest_account, dest_container, dest_object) try: container_info = get_container_info( req.environ, self.app, swift_source='CQ') finally: req.environ['PATH_INFO'] = path_info if not container_info or not is_success(container_info['status']): # this will hopefully 404 later return self.app if 'quota-bytes' in container_info.get('meta', {}) and \ 'bytes' in container_info and \ container_info['meta']['quota-bytes'].isdigit(): content_length = (req.content_length or 0) if 'x-copy-from' in req.headers or req.method == 'COPY': if 'x-copy-from' in req.headers: container, obj = check_copy_from_header(req) path = '/%s/%s/%s/%s' % (version, account, container, obj) object_info = get_object_info(req.environ, self.app, path) if not object_info or not object_info['length']: content_length = 0 else: content_length = int(object_info['length']) new_size = int(container_info['bytes']) + content_length if int(container_info['meta']['quota-bytes']) < new_size: return self.bad_response(req, container_info) if 'quota-count' in container_info.get('meta', {}) and \ 'object_count' in container_info and \ container_info['meta']['quota-count'].isdigit(): new_count = int(container_info['object_count']) + 1 if int(container_info['meta']['quota-count']) < new_count: return self.bad_response(req, container_info) return self.app
def test_get_object_info_swift_source(self): app = FakeApp() req = Request.blank("/v1/a/c/o", environ={"swift.cache": FakeCache()}) get_object_info(req.environ, app, swift_source="LU") self.assertEqual(app.sources, ["LU"])
def __call__(self, req): try: vrs, acc, con, obj = req.split_path(2, 4, rest_with_last=True) except ValueError: return self.app # All requests if req.method == 'GET': account_info = get_account_info(req.environ, self.app) if account_info: recycled = account_info['meta'].get('recycled', '') delete_date = account_info['meta'].get('earliest-delete-date', '') if recycled == 'yes' and delete_date != '': return swob.HTTPNotFound( headers={ 'x-account-meta-recycled': 'yes', 'x-account-meta-earliest-delete-date': delete_date }, body= ("Account is marked for deletion. " "Send X-Remove-Account-Meta-Recycled header via POST to undelete." )) # Account specific requests if con is None: if req.method == 'DELETE': account_info = get_account_info(req.environ, self.app) if account_info: try: recycled = account_info['meta'].get('recycled', '') delete_date = int(account_info['meta'].get( 'earliest-delete-date', '0')) if recycled != "yes": return swob.HTTPMethodNotAllowed( content_type="text/plain", body= ("Account cannot be deleted directly. " "Send 'X-Account-Meta-Recycled: yes' in POST request to mark for deletion.\n" )) if time() < delete_date: return swob.HTTPMethodNotAllowed( content_type="text/plain", headers={ 'x-account-meta-recycled': 'yes', 'x-account-meta-earliest-delete-date': delete_date }, body= ("Account cannot be deleted yet, " "X-Account-Meta-Earliest-Delete-Date not reached yet.\n" )) return self.app except ValueError: return swob.HTTPInternalError( content_type="text/plain", body=( "Internal error. Cannot read recycled state.\n" )) if req.method == 'POST': if 'x-account-meta-earliest-delete-date' in req.headers or 'x-remove-account-meta-earliest-delete-date' in req.headers: return swob.HTTPMethodNotAllowed( content_type="text/plain", body=("Header X-Account-Meta-Earliest-Delete-Date " "cannot be set manually.\n")) if 'x-account-meta-recycled' in req.headers and req.headers[ 'x-account-meta-recycled'] == "yes": req.headers['x-account-meta-recycled'] = "yes" req.headers['x-account-meta-earliest-delete-date'] = str( int(time()) + self.account_recycled_seconds) return self.app if 'x-remove-account-meta-recycled' in req.headers: req.headers['x-remove-account-meta-recycled'] = "x" req.headers[ 'x-remove-account-meta-earliest-delete-date'] = "x" return self.app return self.app # Container specific requests if obj is None: return self.app # Object specific requests if req.method == 'GET': object_info = get_object_info(req.environ, self.app) if object_info: recycled = object_info['meta'].get('recycled', '') delete_date = object_info['meta'].get('delete-date', '') if recycled == 'yes': return swob.HTTPNotFound( headers={ 'x-object-meta-recycled': 'yes', 'x-object-meta-delete-date': delete_date }, body= ("Object is marked for deletion. " "Send X-Remove-Object-Meta-Recycled header via POST to undelete.\n" )) if req.method == 'DELETE': return swob.HTTPMethodNotAllowed( content_type="text/plain", body=( "DELETE requests are not allowed. " "Use POST with 'X-Object-Meta-Recycled: yes' instead.\n")) if req.method == 'POST' or req.method == 'PUT': if 'x-delete-at' in req.headers or 'x-delete-after' in req.headers or 'x-object-meta-delete-date' in req.headers: return swob.HTTPMethodNotAllowed( content_type="text/plain", body= ("Setting X-Delete-At/X-Delete-After/X-Object-Meta-Delete-Date directly is not allowed. " "Use POST with 'X-Object-Meta-Recycled: yes' instead.\n")) if 'x-object-meta-recycled' in req.headers: if req.headers['x-object-meta-recycled'] != "yes": return swob.HTTPBadRequest( content_type="text/plain", body=("Invalid value for X-Object-Meta-Recycled. " "Only 'yes' is allowed.\n")) req.headers['x-object-meta-recycled'] = "yes" req.headers['x-object-meta-delete-date'] = str( int(time()) + self.object_recycle_keep_seconds) req.headers['x-delete-after'] = str( self.object_recycle_keep_seconds) return self.app if 'x-remove-object-meta-recycled' in req.headers: req.headers['x-remove-object-meta-recycled'] = "x" req.headers['x-remove-object-meta-delete-date'] = "x" req.headers['x-remove-delete-at'] = "x" req.headers['x-remove-delete-after'] = "x" return self.app return self.app
def __call__(self, request): if request.method not in ("POST", "PUT", "COPY"): return self.app try: ver, account, container, obj = request.split_path( 2, 4, rest_with_last=True) except ValueError: return self.app if not container: # account request, so we pay attention to the quotas new_quota = request.headers.get( 'X-Account-Meta-Quota-Bytes') remove_quota = request.headers.get( 'X-Remove-Account-Meta-Quota-Bytes') else: # container or object request; even if the quota headers are set # in the request, they're meaningless new_quota = remove_quota = None if remove_quota: new_quota = 0 # X-Remove dominates if both are present if request.environ.get('reseller_request') is True: if new_quota and not new_quota.isdigit(): return HTTPBadRequest() return self.app # deny quota set for non-reseller if new_quota is not None: return HTTPForbidden() if obj and request.method == "POST" or not obj: return self.app if request.method == 'COPY': copy_from = container + '/' + obj else: copy_from = request.headers.get('X-Copy-From') content_length = (request.content_length or 0) account_info = get_account_info(request.environ, self.app) if not account_info or not account_info['bytes']: return self.app try: quota = int(account_info['meta'].get('quota-bytes', -1)) except ValueError: return self.app if quota < 0: return self.app if obj and copy_from: path = '/' + ver + '/' + account + '/' + copy_from.lstrip('/') object_info = get_object_info(request.environ, self.app, path) if not object_info or not object_info['length']: content_length = 0 else: content_length = int(object_info['length']) new_size = int(account_info['bytes']) + content_length if quota < new_size: return HTTPRequestEntityTooLarge() return self.app
def __call__(self, request): if request.method not in ("POST", "PUT", "COPY"): return self.app try: ver, account, container, obj = request.split_path( 2, 4, rest_with_last=True) except ValueError: return self.app if not container: # account request, so we pay attention to the quotas new_quota = request.headers.get( 'X-Account-Meta-Quota-Bytes') remove_quota = request.headers.get( 'X-Remove-Account-Meta-Quota-Bytes') else: # container or object request; even if the quota headers are set # in the request, they're meaningless new_quota = remove_quota = None if remove_quota: new_quota = 0 # X-Remove dominates if both are present if request.environ.get('reseller_request') is True: if new_quota and not new_quota.isdigit(): return HTTPBadRequest() return self.app # deny quota set for non-reseller if new_quota is not None: return HTTPForbidden() if request.method == "POST" or not obj: return self.app if request.method == 'COPY': copy_from = container + '/' + obj else: if 'x-copy-from' in request.headers: src_cont, src_obj = check_copy_from_header(request) copy_from = "%s/%s" % (src_cont, src_obj) else: copy_from = None content_length = (request.content_length or 0) account_info = get_account_info(request.environ, self.app) if not account_info or not account_info['bytes']: return self.app try: quota = int(account_info['meta'].get('quota-bytes', -1)) except ValueError: return self.app if quota < 0: return self.app if copy_from: path = '/' + ver + '/' + account + '/' + copy_from object_info = get_object_info(request.environ, self.app, path) if not object_info or not object_info['length']: content_length = 0 else: content_length = int(object_info['length']) new_size = int(account_info['bytes']) + content_length if quota < new_size: resp = HTTPRequestEntityTooLarge(body='Upload exceeds quota.') if 'swift.authorize' in request.environ: orig_authorize = request.environ['swift.authorize'] def reject_authorize(*args, **kwargs): aresp = orig_authorize(*args, **kwargs) if aresp: return aresp return resp request.environ['swift.authorize'] = reject_authorize else: return resp return self.app
def _test_transient_sysmeta_replaced_by_PUT_or_POST(self, app): # check transient_sysmeta is replaced en-masse by a POST path = '/v1/a/c/o' env = {'REQUEST_METHOD': 'PUT'} hdrs = dict(self.original_transient_sysmeta_headers_1) hdrs.update(self.original_transient_sysmeta_headers_2) hdrs.update(self.original_meta_headers_1) req = Request.blank(path, environ=env, headers=hdrs, body='x') resp = req.get_response(app) self._assertStatus(resp, 201) req = Request.blank(path, environ={}) resp = req.get_response(app) self._assertStatus(resp, 200) self._assertInHeaders(resp, self.original_transient_sysmeta_headers_1) self._assertInHeaders(resp, self.original_transient_sysmeta_headers_2) self._assertInHeaders(resp, self.original_meta_headers_1) info = get_object_info(req.environ, app) self.assertEqual(2, len(info.get('transient_sysmeta', ()))) self.assertEqual({'testa': 'A', 'testb': 'B'}, info['transient_sysmeta']) # POST will replace all existing transient_sysmeta and usermeta values env = {'REQUEST_METHOD': 'POST'} hdrs = dict(self.changed_transient_sysmeta_headers) hdrs.update(self.new_transient_sysmeta_headers_1) req = Request.blank(path, environ=env, headers=hdrs) resp = req.get_response(app) self._assertStatus(resp, 202) req = Request.blank(path, environ={}) resp = req.get_response(app) self._assertStatus(resp, 200) self._assertInHeaders(resp, self.changed_transient_sysmeta_headers) self._assertInHeaders(resp, self.new_transient_sysmeta_headers_1) self._assertNotInHeaders(resp, self.original_meta_headers_1) self._assertNotInHeaders(resp, self.original_transient_sysmeta_headers_2) info = get_object_info(req.environ, app) self.assertEqual(2, len(info.get('transient_sysmeta', ()))) self.assertEqual({'testa': 'changed_A', 'testc': 'C'}, info['transient_sysmeta']) # subsequent PUT replaces all transient_sysmeta and usermeta values env = {'REQUEST_METHOD': 'PUT'} hdrs = dict(self.new_transient_sysmeta_headers_2) hdrs.update(self.original_meta_headers_2) req = Request.blank(path, environ=env, headers=hdrs, body='x') resp = req.get_response(app) self._assertStatus(resp, 201) req = Request.blank(path, environ={}) resp = req.get_response(app) self._assertStatus(resp, 200) self._assertInHeaders(resp, self.original_meta_headers_2) self._assertInHeaders(resp, self.new_transient_sysmeta_headers_2) # meta from previous POST should have gone away... self._assertNotInHeaders(resp, self.changed_transient_sysmeta_headers) self._assertNotInHeaders(resp, self.new_transient_sysmeta_headers_1) # sanity check that meta from first PUT did not re-appear... self._assertNotInHeaders(resp, self.original_meta_headers_1) self._assertNotInHeaders(resp, self.original_transient_sysmeta_headers_1) self._assertNotInHeaders(resp, self.original_transient_sysmeta_headers_2) info = get_object_info(req.environ, app) self.assertEqual(1, len(info.get('transient_sysmeta', ()))) self.assertEqual({'testd': 'D'}, info['transient_sysmeta'])
def __call__(self, env, start_response): req = Request(env) try: version, account, container, obj = req.split_path(4, 4, True) except ValueError: return self.app(env, start_response) container_info = get_container_info( req.environ, self.app, swift_source='ImageScalerMiddleware') # parse query string if req.query_string: qs = parse_qs(req.query_string) if 'size' in qs: req_size = qs['size'] else: self.logger.debug("image-scaler: No image scaling requested.") return self.app(env, start_response) else: # nothing for us to do, no scaling requested self.logger.debug("image-scaler: No image scaling requested.") return self.app(env, start_response) # check container whether scaling is allowed meta = container_info['meta'] if not meta.has_key('image-scaling') or \ meta.has_key('image-scaling') and \ not meta['image-scaling'].lower() in ['true', '1']: # nothing for us to do self.logger.debug("image-scaler: Image scaling not " "allowed. Nothing for us to do.") return self.app(env, start_response) # default allowed extensions allowed_exts = self.conf.get('formats', 'jpg;png;gif') allowed_exts = allowed_exts.lower() allowed_exts = allowed_exts.split(';') # check whether file has the allowed ending if meta.has_key('image-scaling-extensions'): allowed_exts = meta['image-scaling-extensions'].split(',') requested_ext = req.path.rsplit('.', 1)[-1] if not requested_ext.lower() in map(lambda x: x.lower(), allowed_exts): self.logger.info("image-scaler: extension %s not allowed" " for image scaling" % requested_ext) return self.app(env, start_response) # get maxsize from config, otherwise 20 MB max_size = self.conf.get('maxsize', '20971520') try: max_size = int(max_size) except ValueError: max_size = 20971520 self.logger.error( "wrong format for max_size from configuration file, using 20 MB" ) if meta.has_key('image-scaling-max-size'): max_size = int(meta['image-scaling-max-size']) obj_info = get_object_info(req.environ, self.app, swift_source="ImageScalerMiddleware") if int(obj_info['length']) > max_size: self.logger.info("image-scaler: object too large") return self.app(env, start_response) response = ImageScalerResponse(start_response, req_size, self.logger) app_iter = self.app(env, response.scaler_start_response) if app_iter is not None: response.finish_response(app_iter) return response.write()
def PUT(self, req): """HTTP PUT request handler.""" container_info = self.container_info(self.account_name, self.container_name, req) req.acl = container_info['write_acl'] req.environ['swift_sync_key'] = container_info['sync_key'] # is request authorized if 'swift.authorize' in req.environ: aresp = req.environ['swift.authorize'](req) if aresp: return aresp self.enforce_versioning(req) old_slo_manifest = None old_slo_manifest_etag = None # If versioning is disabled, we must check if the object exists. # If it's a NEW SLO (we must check it is not the same manifest), # we will have to delete the parts if the current # operation is a success. if (self.app.delete_slo_parts and not config_true_value( container_info.get('sysmeta', {}).get('versions-enabled', False))): try: dest_info = get_object_info(req.environ, self.app) if 'slo-size' in dest_info['sysmeta']: manifest_env = req.environ.copy() manifest_env['QUERY_STRING'] = 'multipart-manifest=get' manifest_req = make_subrequest(manifest_env, 'GET') manifest_resp = manifest_req.get_response(self.app) old_slo_manifest = json.loads(manifest_resp.body) old_slo_manifest_etag = dest_info.get('etag') except Exception as exc: self.app.logger.warn( ('Failed to check existence of %s. If ' 'overwriting a SLO, old parts may ' 'remain. Error was: %s') % (req.path, exc)) self._update_content_type(req) req.ensure_x_timestamp() # check constraints on object name and request headers error_response = check_object_creation(req, self.object_name) or \ check_content_type(req) if error_response: return error_response if req.headers.get('Oio-Copy-From'): return self._link_object(req) data_source = req.environ['wsgi.input'] if req.content_length: data_source = ExpectedSizeReader(data_source, req.content_length) headers = self._prepare_headers(req) with closing_if_possible(data_source): resp = self._store_object(req, data_source, headers) if (resp.is_success and old_slo_manifest and resp.etag != old_slo_manifest_etag): self.app.logger.debug( 'Previous object %s was a different SLO, deleting parts', req.path) self._delete_slo_parts(req, old_slo_manifest) return resp
def __call__(self, request): if request.method not in ("POST", "PUT", "COPY"): return self.app try: ver, account, container, obj = request.split_path( 2, 4, rest_with_last=True) except ValueError: return self.app if not container: # account request, so we pay attention to the quotas new_quota = request.headers.get('X-Account-Meta-Quota-Bytes') remove_quota = request.headers.get( 'X-Remove-Account-Meta-Quota-Bytes') else: # container or object request; even if the quota headers are set # in the request, they're meaningless new_quota = remove_quota = None if remove_quota: new_quota = 0 # X-Remove dominates if both are present if request.environ.get('reseller_request') is True: if new_quota and not new_quota.isdigit(): return HTTPBadRequest() return self.app # deny quota set for non-reseller if new_quota is not None: return HTTPForbidden() if request.method == "POST" or not obj: return self.app if request.method == 'COPY': copy_from = container + '/' + obj else: if 'x-copy-from' in request.headers: src_cont, src_obj = check_copy_from_header(request) copy_from = "%s/%s" % (src_cont, src_obj) else: copy_from = None content_length = (request.content_length or 0) account_info = get_account_info(request.environ, self.app) if not account_info or not account_info['bytes']: return self.app try: quota = int(account_info['meta'].get('quota-bytes', -1)) except ValueError: return self.app if quota < 0: return self.app if copy_from: path = '/' + ver + '/' + account + '/' + copy_from object_info = get_object_info(request.environ, self.app, path) if not object_info or not object_info['length']: content_length = 0 else: content_length = int(object_info['length']) new_size = int(account_info['bytes']) + content_length if quota < new_size: resp = HTTPRequestEntityTooLarge(body='Upload exceeds quota.') if 'swift.authorize' in request.environ: orig_authorize = request.environ['swift.authorize'] def reject_authorize(*args, **kwargs): aresp = orig_authorize(*args, **kwargs) if aresp: return aresp return resp request.environ['swift.authorize'] = reject_authorize else: return resp return self.app