Beispiel #1
0
    def __call__(self, env, start_response):
        if config_true_value(env.get('swift.crypto.override')):
            return self.app(env, start_response)

        req = Request(env)

        if self.disable_encryption and req.method in ('PUT', 'POST'):
            return self.app(env, start_response)
        try:
            req.split_path(4, 4, True)
        except ValueError:
            return self.app(env, start_response)

        fetch_crypto_keys = env.get(CRYPTO_KEY_CALLBACK)
        if fetch_crypto_keys is not None:
            try:
                fetch_crypto_keys()
            except HTTPException as exc:
                if MISSING_KEY_MSG in exc.body:
                    if req.method in ('PUT', 'POST'):
                        # No key, just upload without encryption
                        env['swift.crypto.override'] = True
                    # else:
                    #   let the thing fail later,
                    #   if a key is required for decoding
                else:
                    raise
            except Exception:
                # Let the parent class handle other exceptions
                pass
        res = super(Encrypter, self).__call__(env, start_response)
        return res
Beispiel #2
0
    def __call__(self, env, start_response):
        # If override is set in env, then just pass along
        if config_true_value(env.get('swift.crypto.override')):
            return self.app(env, start_response)

        req = Request(env)

        if self.disable_encryption and req.method in ('PUT', 'POST'):
            return self.app(env, start_response)
        try:
            req.split_path(4, 4, True)
        except ValueError:
            return self.app(env, start_response)

        if req.method in ('GET', 'HEAD'):
            handler = EncrypterObjContext(self, self.logger).handle_get_or_head
        elif req.method == 'PUT':
            handler = EncrypterObjContext(self, self.logger).handle_put
        elif req.method == 'POST':
            handler = EncrypterObjContext(self, self.logger).handle_post
        else:
            # anything else
            return self.app(env, start_response)

        try:
            return handler(req, start_response)
        except HTTPException as err_resp:
            return err_resp(env, start_response)
Beispiel #3
0
    def __call__(self, env, start_response):
        self.logger.debug('Initialising gate middleware')

        req = Request(env)
        try:
            version, account = req.split_path(1, 3, True)
        except ValueError:
            return HttpNotFound(request=req)
        
        if account is 'gate':
            # Handles direct calls to gate
            return HttpOk

        if 'X-Gate-Verify' in env:
            verify = env['X-Gate-Verify']
            self.logger.debug('Verification request: %s algorithms: %s' % (req.path, verify))

            try:
                version, account, container, obj = req.split_path(4, 4, True)
            except ValueError:
                return HTTPBadRequest(request=req)

            algorithms = verify.split(',')
            for algo in algorithms:
                metakey = 'X-Object-Meta-Gate-%s' % algo.upper()
                if metakey not in env:
                    self.logger.debug('Invalid verification request, object missing: %s' % (metakey))
                    return HTTPBadRequest(request=req)

            if publish_verify(req.path, algorithms):
                for algo in algorithms:
                    statuskey = 'X-Object-Meta-Gate-Verify-%s-Status' % algo.upper()
                    env[statuskey] = 'Queued'
                    env['X-Object-Meta-Gate-Verify'] = verify

        if 'X-Gate-Process' in env:
            module = env['X-Gate-Process']
            self.logger.debug('Process request: %s module: %s' % (req.path, module))

            try:
                version, case, container, obj = req.split_path(4, 4, True)
            except ValueError:
                return HTTPBadRequest(request=req)

            if publish_process(req.path, algorithms):
                for algo in algorithms:
                    env['X-Object-Meta-Gate-Process'] = module
                    env['X-Object-Meta-Gate-Process-Status'] = 'Queued'

        # TODO: Get reponse to see if a fake object
        reponse = self.app(env, start_response)
        return reponse
Beispiel #4
0
    def __call__(self, env, start_response):
        """
        WSGI entry point
        """
        req = Request(env)
        try:
            vrs, account, container, obj = req.split_path(4, 4, True)
        except ValueError:
            return self.app(env, start_response)

        # install our COPY-callback hook
        env["swift.copy_hook"] = self.copy_hook(
            env.get("swift.copy_hook", lambda src_req, src_resp, sink_req: src_resp)
        )

        try:
            if req.method == "PUT" and req.params.get("multipart-manifest") == "put":
                return self.handle_multipart_put(req, start_response)
            if req.method == "DELETE" and req.params.get("multipart-manifest") == "delete":
                return self.handle_multipart_delete(req)(env, start_response)
            if req.method == "GET" or req.method == "HEAD":
                return self.handle_multipart_get_or_head(req, start_response)
            if "X-Static-Large-Object" in req.headers:
                raise HTTPBadRequest(
                    request=req,
                    body="X-Static-Large-Object is a reserved header. "
                    "To create a static large object add query param "
                    "multipart-manifest=put.",
                )
        except HTTPException as err_resp:
            return err_resp(env, start_response)

        return self.app(env, start_response)
Beispiel #5
0
    def __call__(self, env, start_response):
        """
        WSGI entry point.
        Wraps env in swob.Request object and passes it down.

        :param env: WSGI environment dictionary
        :param start_response: WSGI callable
        """
        req = Request(env)
        if self.memcache_client is None:
            # 在pipline中memcache middleware应该在前面
            # memcache为每一个请求建立一个memcache client,并把这个client放入env中
            self.memcache_client = cache_from_env(env)
        if not self.memcache_client:
            self.logger.warning(
                _('Warning: Cannot ratelimit without a memcached client'))
            return self.app(env, start_response)
        try:
            version, account, container, obj = req.split_path(1, 4, True)
        except ValueError:
            return self.app(env, start_response)
        ratelimit_resp = self.handle_ratelimit(req, account, container, obj)
        # ratelimit_resp 为 None 或者 Response 对象,Response 对象是可调用的。
        # ratelimit 操作正常执行,或者不需要执行,返回 None,接着向下调用。
        if ratelimit_resp is None:
            return self.app(env, start_response)
        else:
            # ratelimit 执行失败,或者需要返回错误信息,返回 Response 对象。
            # 不用向下调用,直接返回。
            return ratelimit_resp(env, start_response)
    def __call__(self, env, start_response):
        # making a duplicate, because if this is a COPY request, we will
        # modify the PATH_INFO to find out if the 'Destination' is in a
        # versioned container
        req = Request(env.copy())
        try:
            (version, account, container, obj) = req.split_path(3, 4, True)
        except ValueError:
            return self.app(env, start_response)

        # In case allow_versioned_writes is set in the filter configuration,
        # the middleware becomes the authority on whether object
        # versioning is enabled or not. In case it is not set, then
        # the option in the container configuration is still checked
        # for backwards compatibility

        # For a container request, first just check if option is set,
        # can be either true or false.
        # If set, check if enabled when actually trying to set container
        # header. If not set, let request be handled by container server
        # for backwards compatibility.
        # For an object request, also check if option is set (either T or F).
        # If set, check if enabled when checking versions container in
        # sysmeta property. If it is not set check 'versions' property in
        # container_info
        allow_versioned_writes = self.conf.get('allow_versioned_writes')
        if allow_versioned_writes and container and not obj:
            return self.container_request(req, start_response,
                                          allow_versioned_writes)
        elif obj and req.method in ('PUT', 'COPY', 'DELETE'):
            return self.object_request(
                req, version, account, container, obj,
                allow_versioned_writes)(env, start_response)
        else:
            return self.app(env, start_response)
Beispiel #7
0
    def __call__(self, env, start_response):
        """
        WSGI entry point
        """
        req = Request(env)
        try:
            vrs, account, container, obj = req.split_path(4, 4, True)
        except ValueError:
            return self.app(env, start_response)

        # install our COPY-callback hook
        env['swift.copy_hook'] = self.copy_hook(
            env.get('swift.copy_hook',
                    lambda src_req, src_resp, sink_req: src_resp))

        try:
            if req.method == 'PUT' and \
                    req.params.get('multipart-manifest') == 'put':
                return self.handle_multipart_put(req, start_response)
            if req.method == 'DELETE' and \
                    req.params.get('multipart-manifest') == 'delete':
                return self.handle_multipart_delete(req)(env, start_response)
            if req.method == 'GET' or req.method == 'HEAD':
                return self.handle_multipart_get_or_head(req, start_response)
            if 'X-Static-Large-Object' in req.headers:
                raise HTTPBadRequest(
                    request=req,
                    body='X-Static-Large-Object is a reserved header. '
                    'To create a static large object add query param '
                    'multipart-manifest=put.')
        except HTTPException as err_resp:
            return err_resp(env, start_response)

        return self.app(env, start_response)
Beispiel #8
0
    def __call__(self, env, start_response):
        req = Request(env)

        try:
            parts = req.split_path(2, 4, True)
        except ValueError:
            return self.app(env, start_response)

        if req.method in ('PUT', 'POST', 'GET', 'HEAD'):
            # handle only those request methods that may require keys
            km_context = KeyMasterContext(self, req, *parts[1:])
            try:
                return km_context.handle_request(req, start_response)
            except HTTPException as err_resp:
                return err_resp(env, start_response)
            except KeyError as err:
                if 'object' in err.args:
                    self.app.logger.debug(
                        'Missing encryption key, cannot handle request')
                    raise HTTPBadRequest(MISSING_KEY_MSG)
                else:
                    raise

        # anything else
        return self.app(env, start_response)
Beispiel #9
0
    def __call__(self, env, start_response):
        """
        WSGI entry point
        """
        req = Request(env)
        try:
            vrs, account, container, obj = req.split_path(1, 4, True)
        except ValueError:
            return self.app(env, start_response)
        try:
            if obj:
                if req.method == 'PUT' and \
                        req.params.get('multipart-manifest') == 'put':
                    return self.handle_multipart_put(req, start_response)
                if req.method == 'DELETE' and \
                        req.params.get('multipart-manifest') == 'delete':
                    return self.handle_multipart_delete(req)(env,
                                                             start_response)
                if 'X-Static-Large-Object' in req.headers:
                    raise HTTPBadRequest(
                        request=req,
                        body='X-Static-Large-Object is a reserved header. '
                        'To create a static large object add query param '
                        'multipart-manifest=put.')
        except HTTPException as err_resp:
            return err_resp(env, start_response)

        return self.app(env, start_response)
Beispiel #10
0
    def __call__(self, env, start_response):
        """
        WSGI entry point
        """
        req = Request(env)
        try:
            vrs, account, container, obj = req.split_path(4, 4, True)
            if DEBUG:
                print('obj:%s' % obj)
        except ValueError:
            return self.app(env, start_response)
        try:
            if env['REQUEST_METHOD'] == "PUT":
                # Read the object content and enc_key in memory 
                # encrypt the object with the enc_key
                content = env['wsgi.input'].read()
                if content:
                    return Response(status=403,
                                body="content %s detected" % content,
                                content_type="text/plain")(env, start_response)
                else:
                    return Response(status=403,
                                body="content not detected",
                                content_type="text/plain")(env, start_response)
                if DEBUG:
                    print(content)
                #encryptor = Encryptor()
                content = content + "yyyyyyyyyyyyyyyyy"
            # Maybe add the decrytion process later 
            #elif env['REQUEST_METHOD'] == "GET":
        except HTTPException as err_resp:
            return err_resp(env, start_response)

        return self.app(env, start_response)
Beispiel #11
0
    def __call__(self, env, start_response):
        req = Request(env)
        try:
            (version, account, container, obj) = req.split_path(4, 4, True)
        except ValueError:
            # If obj component is not present in req, do not proceed further.
            return self.app(env, start_response)

        try:
            # In some cases, save off original request method since it gets
            # mutated into PUT during handling. This way logging can display
            # the method the client actually sent.
            if req.method == 'PUT' and req.headers.get('X-Copy-From'):
                return self.handle_PUT(req, start_response)
            elif req.method == 'COPY':
                req.environ['swift.orig_req_method'] = req.method
                return self.handle_COPY(req, start_response,
                                        account, container, obj)
            elif req.method == 'OPTIONS':
                # Does not interfere with OPTIONS response from
                # (account,container) servers and /info response.
                return self.handle_OPTIONS(req, start_response)

        except HTTPException as e:
            return e(req.environ, start_response)

        return self.app(env, start_response)
Beispiel #12
0
    def __call__(self, env, start_response):
        req = Request(env)
        try:
            (version, account, container, obj) = req.split_path(4, 4, True)
        except ValueError:
            # If obj component is not present in req, do not proceed further.
            return self.app(env, start_response)

        self.account_name = account
        self.container_name = container
        self.object_name = obj

        # Save off original request method (COPY/POST) in case it gets mutated
        # into PUT during handling. This way logging can display the method
        # the client actually sent.
        req.environ['swift.orig_req_method'] = req.method

        if req.method == 'PUT' and req.headers.get('X-Copy-From'):
            return self.handle_PUT(req, start_response)
        elif req.method == 'COPY':
            return self.handle_COPY(req, start_response)
        elif req.method == 'POST' and self.object_post_as_copy:
            return self.handle_object_post_as_copy(req, start_response)
        elif req.method == 'OPTIONS':
            # Does not interfere with OPTIONS response from (account,container)
            # servers and /info response.
            return self.handle_OPTIONS(req, start_response)

        return self.app(env, start_response)
Beispiel #13
0
    def __call__(self, env, start_response):
        req = Request(env)
        # We want to check POST here
        # it can possibly have content_length > 0
        if not self.enforce_quota or req.method not in ("POST", "PUT", "COPY"):
            return self.app(env, start_response)
        account_info = get_account_info(req.environ, self.app,
                                        swift_source='litequota')
        if not account_info:
            return self.app(env, start_response)
        service_plan = assemble_from_partial(self.metadata_key,
                                             account_info['meta'])
        try:
            ver, account, container, obj = \
                req.split_path(2, 4, rest_with_last=True)
        except ValueError:
            return self.app(env, start_response)
        if not service_plan and req.method == 'PUT' and not obj:
            service_plan = self.set_serviceplan(req, account)
        if not service_plan:
            return self.app(env, start_response)
        try:
            service_plan = json.loads(service_plan)
        except ValueError:
            return self.app(env, start_response)

        if service_plan.get('storage', None):
            resp = self.apply_storage_quota(req, service_plan['storage'],
                                            account_info,
                                            ver, account, container, obj)
            if resp:
                return resp(env, start_response)
        return self.app(env, start_response)
Beispiel #14
0
    def __call__(self, env, start_response):
        """
        WSGI entry point.
        Wraps env in swob.Request object and passes it down.

        :param env: WSGI environment dictionary
        :param start_response: WSGI callable
        """
        req = Request(env)
        if self.memcache_client is None:
            self.memcache_client = cache_from_env(env)
        if not self.memcache_client:
            self.logger.warning(
                _('Warning: Cannot ratelimit without a memcached client'))
            return self.app(env, start_response)
        try:
            version, account, container, obj = req.split_path(1, 4, True)
        except ValueError:
            return self.app(env, start_response)
        if not valid_api_version(version):
            return self.app(env, start_response)
        ratelimit_resp = self.handle_ratelimit(req, account, container, obj)
        if ratelimit_resp is None:
            return self.app(env, start_response)
        else:
            return ratelimit_resp(env, start_response)
Beispiel #15
0
    def __call__(self, env, start_response):
        """
        WSGI entry point
        """
        req = Request(env)
        try:
            vrs, account, container, obj = req.split_path(4, 4, True)
        except ValueError:
            return self.app(env, start_response)

        # install our COPY-callback hook
        env['swift.copy_hook'] = self.copy_hook(
            env.get('swift.copy_hook',
                    lambda src_req, src_resp, sink_req: src_resp))

        if ((req.method == 'GET' or req.method == 'HEAD') and
                req.params.get('multipart-manifest') != 'get'):
            return GetContext(self, self.logger).\
                handle_request(req, start_response)
        elif req.method == 'PUT':
            error_response = self.validate_x_object_manifest_header(
                req, start_response)
            if error_response:
                return error_response(env, start_response)
        return self.app(env, start_response)
Beispiel #16
0
    def get_creds(self, environ):
        req = Request(environ)
        try:
            parts = req.split_path(1, 4, True)
            _, account, _, _ = parts
        except ValueError:
            account = None

        env_identity = self._integral_keystone_identity(environ)
        if not env_identity:
            # user identity is not confirmed. (anonymous?)
            creds = {
                'identity': None,
                'is_authoritative': (account and
                                     account.startswith(self.reseller_prefix))
            }
            return creds

        tenant_id, tenant_name = env_identity['tenant']
        user_id, user_name = env_identity['user']
        roles = [r.strip() for r in env_identity.get('roles', [])]
        account = self._get_account_for_tenant(tenant_id)
        is_admin = (tenant_name == user_name)

        creds = {
            "identity": env_identity,
            "roles": roles,
            "account": account,
            "tenant_id": tenant_id,
            "tenant_name": tenant_name,
            "user_id": user_id,
            "user_name": user_name,
            "is_admin": is_admin
        }
        return creds
    def __call__(self, env, start_response):
        self.logger.debug('Calling Utilization Middleware')

        req = Request(env)
        if self.check_api_call(env):
            return self.GET(req)(env, start_response)

        try:
            version, account, container, obj = req.split_path(2, 4, True)
        except ValueError:
            return self.app(env, start_response)

        remote_user = env.get('REMOTE_USER')
        if not remote_user or (isinstance(remote_user, basestring) and
                               remote_user.startswith('.wsgi')):
            self.logger.debug('### SKIP: REMOTE_USER is %s' % remote_user)
            return self.app(env, start_response)

        start_response_args = [None]
        input_proxy = InputProxy(env['wsgi.input'])
        env['wsgi.input'] = input_proxy

        def my_start_response(status, headers, exc_info=None):
            start_response_args[0] = (status, list(headers), exc_info)

        def iter_response(iterable):
            iterator = iter(iterable)
            try:
                chunk = next(iterator)
                while not chunk:
                    chunk = next(iterator)
            except StopIteration:
                chunk = ''

            if start_response_args[0]:
                start_response(*start_response_args[0])

            bytes_sent = 0
            try:
                while chunk:
                    bytes_sent += len(chunk)
                    yield chunk
                    chunk = next(iterator)
            finally:
                try:
                    self.publish_sample(env, account,
                                        input_proxy.bytes_received,
                                        bytes_sent)
                except Exception:
                    self.logger.exception('Failed to publish samples')

        try:
            iterable = self.app(env, my_start_response)
        except Exception:
            self.publish_sample(env, account, input_proxy.bytes_received, 0)
            raise
        else:
            return iter_response(iterable)
Beispiel #18
0
    def __call__(self, env, start_response):
        req = Request(env)
        try:
            (version, account, container, obj) = req.split_path(4, 4, True)
        except ValueError:
            # If obj component is not present in req, do not proceed further.
            return self.app(env, start_response)

        self.version = version
        self.account_name = account

        try:
            # Check if fastcopy is possible
            # only plain object as source and destination
            # FIXME: handle COPY method (with Destination-* headers)
            check, source_resp = self.fast_copy_allowed(req)
            if check:
                self.logger.debug("COPY: fast copy allowed")
                env['HTTP_OIO_COPY_FROM'] = unquote(env['HTTP_X_COPY_FROM'])
                del env['HTTP_X_COPY_FROM']
                # handle metadata
                if not config_true_value(req.headers.get('x-fresh-metadata',
                                                         'false')):

                    # cf copy middlware
                    exclude_headers = ('x-static-large-object', 'etag',
                                       'x-object-manifest', 'content-type',
                                       'x-timestamp', 'x-backend-timestamp')
                    # copy original headers but don't overwrite source headers
                    copy_header_subset(
                        source_resp, req,
                        lambda k: k.lower() not in exclude_headers
                        and k.lower() not in req.headers)

                def _start_response(status, headers, exc_info=None):
                    headers.append(('X-Copied-From-Account',
                                    env.get('HTTP_X_COPY_FROM_ACCOUNT',
                                            self.account_name)))
                    headers.append(('X-Copied-From',
                                    quote(env['HTTP_OIO_COPY_FROM'])))
                    start_response(status, headers, exc_info)

                return self.app(env, _start_response)
        except HTTPException as exc:
            return exc(req.environ, start_response)

        return super(OioServerSideCopyMiddleware, self).__call__(
            env, start_response)
Beispiel #19
0
    def __call__(self, env, start_response):
        req = Request(env)

        try:
            parts = req.split_path(2, 4, True)
        except ValueError:
            return self.app(env, start_response)

        if req.method in ('PUT', 'POST', 'GET', 'HEAD'):
            # handle only those request methods that may require keys
            km_context = KeyMasterContext(self, *parts[1:])
            try:
                return km_context.handle_request(req, start_response)
            except HTTPException as err_resp:
                return err_resp(env, start_response)

        # anything else
        return self.app(env, start_response)
Beispiel #20
0
    def __call__(self, env, start_response):
        """
        WSGI entry point
        """
        req = Request(env)
        try:
            vrs, account, container, obj = req.split_path(4, 4, True)
        except ValueError:
            return self.app(env, start_response)

        if ((req.method == 'GET' or req.method == 'HEAD') and
                req.params.get('multipart-manifest') != 'get'):
            return GetContext(self, self.logger).\
                handle_request(req, start_response)
        elif req.method == 'PUT':
            error_response = self._validate_x_object_manifest_header(req)
            if error_response:
                return error_response(env, start_response)
        return self.app(env, start_response)
Beispiel #21
0
    def __call__(self, env, start_response):
        req = Request(env)
        try:
            version, acc, cont, obj = req.split_path(3, 4, True)
        except ValueError:
            return self.app(env, start_response)

        try:
            if obj:
                # object context
                context = SymlinkObjectContext(self.app, self.logger,
                                               self.symloop_max)
                return context.handle_object(req, start_response)
            else:
                # container context
                context = SymlinkContainerContext(self.app, self.logger)
                return context.handle_container(req, start_response)
        except HTTPException as err_resp:
            return err_resp(env, start_response)
Beispiel #22
0
    def __call__(self, env, start_response):
        req = Request(env)
        try:
            parts = req.split_path(3, 4, True)
        except ValueError:
            return self.app(env, start_response)

        if parts[3] and req.method in ('GET', 'HEAD'):
            handler = DecrypterObjContext(self, self.logger).handle
        elif parts[2] and req.method == 'GET':
            handler = DecrypterContContext(self, self.logger).handle
        else:
            # url and/or request verb is not handled by decrypter
            return self.app(env, start_response)

        try:
            return handler(req, start_response)
        except HTTPException as err_resp:
            return err_resp(env, start_response)
    def __call__(self, env, start_response):
        request = Request(env)

        if not request.path.startswith(self.endpoint_path):
            return self.app(env, start_response)

        if request.method != 'GET':
            resp = HTTPMethodNotAllowed(req=request, headers={"Allow": "GET"})
            return resp(env, start_response)

        try:
            self.config.read(self.conf["__file__"])
        except (IOError, KeyError):
            resp = HTTPServerError(req=request,
                                   body="An error occurred",
                                   content_type="text/plain")
            return resp(env, start_response)

        config_dict = self._config_parser_to_nested_dict()

        try:
            endpoint, section, option = request.split_path(1, 3, False)
        except ValueError:
            resp = HTTPBadRequest(req=request, headers={})
            return resp(env, start_response)

        try:
            if section and option:
                config_dict = {section: {option: config_dict[section][option]}}
            elif section and not option:
                config_dict = {section: config_dict[section]}
        except KeyError:
            resp = HTTPNotFound(req=request,
                                body="Requested values aren't available",
                                content_type="text/plain")
            return resp(env, start_response)

        resp = Response(req=request,
                        body=json.dumps(config_dict),
                        content_type="application/json")

        return resp(env, start_response)
    def __call__(self, env, start_response):
        request = Request(env)

        if request.method == 'PUT':
            try:
                version, account, container, obj = \
                    request.split_path(1, 4, True)
            except ValueError:
                return self.app(env, start_response)

            # check container creation request
            if account and container and not obj:
                policy_name = request.headers.get('X-Storage-Policy', '')
                default_policy = POLICIES.default.name
                if (policy_name in self.policies) or \
                   (policy_name == '' and default_policy in self.policies):

                    container = unquote(container)
                    if len(container) > constraints. \
                            SOF_MAX_CONTAINER_NAME_LENGTH:
                        resp = HTTPBadRequest(request=request)
                        resp.body = \
                            'Container name length of %d longer than %d' % \
                            (len(container),
                                constraints.SOF_MAX_CONTAINER_NAME_LENGTH)
                        return resp(env, start_response)
            elif account and container and obj:
                # check object creation request
                obj = unquote(obj)

                container_info = get_container_info(
                    env, self.app)
                policy = POLICIES.get_by_index(
                    container_info['storage_policy'])

                if policy.name in self.policies:
                    error_response = sof_check_object_creation(request, obj)
                    if error_response:
                        self.logger.warn("returning error: %s", error_response)
                        return error_response(env, start_response)

        return self.app(env, start_response)
Beispiel #25
0
    def __call__(self, env, start_response):
        req = Request(env)

        if req.method not in self.write_methods:
            return self.app(env, start_response)

        try:
            version, account, container, obj = req.split_path(1, 4, True)
        except ValueError:
            return self.app(env, start_response)

        if req.method == 'COPY' and 'Destination-Account' in req.headers:
            dest_account = req.headers.get('Destination-Account')
            account = check_account_format(req, dest_account)

        if self.account_read_only(req, account):
            msg = 'Writes are disabled for this account.'
            return HTTPMethodNotAllowed(body=msg)(env, start_response)

        return self.app(env, start_response)
Beispiel #26
0
    def get_target(self, environ):
        req = Request(environ)
        try:
            parts = req.split_path(1, 4, True)
            version, account, container, obj = parts
        except ValueError:
            version = account = container = obj = None

        referrers, acls = swift_acl.parse_acl(getattr(req, 'acl', None))
        target = {
            "req": req,
            "method": req.method.lower(),
            "version": version,
            "account": account,
            "container": container,
            "object": obj,
            "acls": acls,
            "referrers": referrers
        }
        return target
Beispiel #27
0
    def __call__(self, env, start_response):
        req = Request(env)
        try:
            (api_version, account, container, obj) = req.split_path(3, 4, True)
        except ValueError:
            return self.app(env, start_response)

        # In case allow_versioned_writes is set in the filter configuration,
        # the middleware becomes the authority on whether object
        # versioning is enabled or not. In case it is not set, then
        # the option in the container configuration is still checked
        # for backwards compatibility

        # For a container request, first just check if option is set,
        # can be either true or false.
        # If set, check if enabled when actually trying to set container
        # header. If not set, let request be handled by container server
        # for backwards compatibility.
        # For an object request, also check if option is set (either T or F).
        # If set, check if enabled when checking versions container in
        # sysmeta property. If it is not set check 'versions' property in
        # container_info
        allow_versioned_writes = self.conf.get('allow_versioned_writes')
        if allow_versioned_writes and container and not obj:
            try:
                return self.container_request(req, start_response,
                                              allow_versioned_writes)
            except HTTPException as error_response:
                return error_response(env, start_response)
        elif (obj and (req.method in ('PUT', 'DELETE') and
                       not req.environ.get('swift.post_as_copy') or
                       req.method in ('HEAD', 'GET', 'POST'))):
            try:
                return self.object_request(
                    req, api_version, account, container, obj,
                    allow_versioned_writes)(env, start_response)
            except HTTPException as error_response:
                return error_response(env, start_response)
        else:
            return self.app(env, start_response)
Beispiel #28
0
 def __call__(self, env, start_response):
     req = Request(env)
     new_service = env.get('liteauth.new_service', None)
     if new_service:
         account_id = env.get('REMOTE_USER', '')
         if not account_id:
             return HTTPInternalServerError()
         if not self.activate_service(account_id, new_service, req.environ):
                 return HTTPInternalServerError()
     if req.method in ['PUT', 'POST'] and not 'x-zerovm-execute' in req.headers:
         account_info = get_account_info(req.environ, self.app,
                                         swift_source='litequota')
         service_plan = assemble_from_partial(self.metadata_key,
                                              account_info['meta'])
         if service_plan:
             try:
                 service_plan = json.loads(service_plan)
                 path_parts = req.split_path(2, 4, rest_with_last=True)
             except ValueError:
                 return self.app(env, start_response)
             if len(path_parts) == 3:
                 quota = service_plan['storage']['containers']
                 new_size = int(account_info['container_count'])
                 if 0 <= quota < new_size:
                     return HTTPRequestEntityTooLarge(
                         body='Over quota: containers')(env, start_response)
             else:
                 new_size = int(account_info['bytes']) + (req.content_length or 0)
                 quota = service_plan['storage']['bytes']
                 if 0 <= quota < new_size:
                     return HTTPRequestEntityTooLarge(
                         body='Over quota: bytes')(env, start_response)
                 quota = service_plan['storage']['objects']
                 new_size = int(account_info['total_object_count'])
                 if 0 <= quota < new_size:
                     return HTTPRequestEntityTooLarge(
                         body='Over quota: objects')(env, start_response)
     return self.app(env, start_response)
    def __call__(self, env, start_response):

        request = Request(env.copy())
        if request.method != "GET":
            return self.app(env, start_response)

        (version, acc, con, obj) = request.split_path(1, 4, False)
        if not obj:
            return self.app(env, start_response)

        memcache_key = 'RequestCache/%s/%s/%s' % (acc, con, obj)

        cached_content = self.memcache.get(memcache_key)
        if cached_content:
            response = Response(request=request, body=cached_content)
            response.headers['X-RequestCache'] = 'True'
            return response(env, start_response)

        sub_req = wsgi.make_subrequest(env)
        sub_resp = sub_req.get_response(self.app)
        self.memcache.set(memcache_key, sub_resp.body, time=86400.0)

        return self.app(env, start_response)
Beispiel #30
0
    def __call__(self, env, start_response):
        req = Request(env)

        # Split request path to determine version, account, container, object
        try:
            (version, account, container, obj) = req.split_path(2, 4, True)
        except ValueError:
            self.logger.debug('split_path exception')
            return self.app(env, start_response)
        self.logger.debug(':%s:%s:%s:%s:', version, account, container, obj)

        # If request is not HLM request or not a GET, it is not processed
        # by this middleware
        method = req.method
        query = req.query_string or ''
        if not (method == 'POST'
                and ('MIGRATE' in query
                     or 'RECALL' in query)
                or method == 'GET'):
            return self.app(env, start_response)

        # Process GET object data request, if object is migrated return error
        # code 412 'Precondition Failed' (consider using 455 'Method Not Valid
        # in This State') - the error code is returned if any object replica is
        # migrated.
        # TODO: provide option to return error code only if all replicas are
        # migrated, and redirect get request to one of non-migrated replicas
        if req.method == "GET" and obj and 'STATUS' not in query:
            # check status and either let GET proceed or return error code
            rc, out, replicas_status = self.get_object_replicas_status(
                req, account, container, obj)
            if rc == REMOTE_STATUS:
                #send the replica status to requester node
                return Response(status=HTTP_OK,
                                body=out,
                                content_type="text/plain")(env,
                                                           start_response)
            self.logger.debug('replicas_status %s', str(replicas_status))
            ret_error = False
            for replica_status in replicas_status:
                status = literal_eval(replica_status)['status']
                if status not in ['resident', 'premigrated']:
                    ret_error = True
            if ret_error:
                return Response(status=HTTP_PRECONDITION_FAILED,
                                body="Object %s needs to be RECALL-ed before "
                                "it can be accessed.\n" %
                                literal_eval(replicas_status[0])['object'],
                                content_type="text/plain")(env, start_response)

            return self.app(env, start_response)

        # Process POST request to migrate/recall object
        elif method == 'POST' and obj:
            if 'MIGRATE' in query or 'RECALL' in query:
                if 'MIGRATE' in query:
                    hlm_req = 'MIGRATE'
                    hlm_backend = self.migrate_backend
                elif 'RECALL' in query:
                    hlm_req = 'RECALL'
                    hlm_backend = self.recall_backend
                # submit hlm request for object replicas
                status, out = self.submit_object_replicas_migration_recall(
                    req, account, container, obj, hlm_req, hlm_backend)
                self.logger.debug('submit_object_replicas_migration_recall()')
                if status == SUBMITTED_FORWARDED_REQUEST:
                    self.logger.debug('SUBMITTED_FORWARDED_REQUEST')
                    return Response(status=HTTP_OK,
                                    body='Accepted remote replica HLM request',
                                    content_type="text/plain")(env,
                                                               start_response)
                elif status == FAILED_SUBMITTING_REQUEST:
                    self.logger.debug('FAILED_SUBMITTING_REQUEST')
                    return Response(status=HTTP_INTERNAL_SERVER_ERROR,
                                    body=out,
                                    content_type="text/plain")(env,
                                                               start_response)
                elif status == SUBMITTED_REQUESTS:
                    self.logger.debug('SUBMITTED_REQUESTS')
                    return Response(status=HTTP_OK,
                                    body='Accepted %s request.\n' % hlm_req,
                                    content_type="text/plain")(env,
                                                               start_response)
                else:  # invalid case
                    self.logger.debug('INVALID_CASE')
                    return Response(status=HTTP_INTERNAL_SERVER_ERROR,
                                    body=out,
                                    content_type="text/plain")(env,
                                                               start_response)

        # Process GET object status request
        elif req.method == "GET" and obj:
            if 'STATUS' in query:
                # Get status of each replica
                rc, out, replicas_status = self.get_object_replicas_status(
                    req, account, container, obj)
                if rc == REMOTE_STATUS:
                    # send the replica status to requester node
                    return Response(status=HTTP_OK,
                                    body=out,
                                    content_type="text/plain")(env,
                                                               start_response)
                # Prepare/format object status info to report
                # (json is default format)
                out = self.format_object_status_info_for_reporting(
                    req, replicas_status) + '\n'
                # Report object status
                return Response(status=HTTP_OK,
                                body=out,
                                content_type="text/plain")(env, start_response)

        # Process container request
        if (container and not obj and
           ((method == 'POST' and ('MIGRATE' in query or 'RECALL' in query))
                or method == 'GET' and 'STATUS' in query)):
            self.logger.debug('Process container request')
            # Get list of objects
            list_url = 'http://%(ip)s:8080%(url)s'
            list_req = list_url % {'ip': self.ip,
                                   'url': req.path}
            self.logger.debug('list_req: %s', list_req)
            self.logger.debug('req.headers: %s', str(req.headers))
            token = req.headers['X-Storage-Token']
            self.logger.debug('token: %s', token)
            headers = {'X-Storage-Token': token}
            response = requests.get(list_req, headers=headers)
            self.logger.debug('response.headers: %s', str(response.headers))
            self.logger.debug('list: %s', str(response.content))
            objects = response.content.strip().split('\n')
            # Submit migration or recall
            if method == 'POST':
                if 'MIGRATE' in query:
                    hlm_req = 'MIGRATE'
                    hlm_backend = self.migrate_backend
                elif 'RECALL' in query:
                    hlm_req = 'RECALL'
                    hlm_backend = self.recall_backend
                # submit hlm requests
                success = 0
                failure = 0
                for obj in objects:
                    self.logger.debug('obj: %s', obj)
                    status, out = self.submit_object_replicas_migration_recall(
                        req, account, container, obj, hlm_req, hlm_backend)
                    self.logger.debug('submit_object_replicas_migr.._recall()')
                    if status == SUBMITTED_FORWARDED_REQUEST:
                        self.logger.debug('SUBMITTED_FORWARDED_REQUEST')
                        return Response(status=HTTP_OK,
                                        body='Accepted remote replica'
                                        'HLM request',
                                        content_type="text/plain")(
                            env, start_response)
                    elif status == FAILED_SUBMITTING_REQUEST:
                        self.logger.debug('FAILED_SUBMITTING_REQUEST')
                        failure += 1
                    elif status == SUBMITTED_REQUESTS:
                        self.logger.debug('SUBMITTED_REQUESTS')
                        success += 1
                if failure == 0:
                    return Response(status=HTTP_OK,
                                    body='Submitted %s requests.\n' % hlm_req,
                                    content_type="text/plain")(env,
                                                               start_response)
                elif success == 0:
                    return Response(status=HTTP_INTERNAL_SERVER_ERROR,
                                    body='Failed to submit %s requests.\n' %
                                    hlm_req,
                                    content_type="text/plain")(env,
                                                               start_response)
                else:
                    return Response(status=HTTP_OK,
                                    body="Submitting %s requests"
                                    " is only partially"
                                    " successful.\n" % hlm_req,
                                    content_type="text/plain")(env,
                                                               start_response)
            elif method == 'GET':
                # submit hlm requests
                accumulated_out = "["
                for obj in objects:
                    self.logger.debug('obj: %s', obj)

                    # Get status of each replica
                    # rewrite req.path to point to object instead of container
                    req_orig_path = req.environ['PATH_INFO']
                    req.environ['PATH_INFO'] += "/" + obj
                    rc, out, replicas_status = self.get_object_replicas_status(
                        req, account, container, obj)

                    # Prepare/format object status info to report
                    # (json is default format)
                    accumulated_out += self.format_object_status_info_for_reporting(
                        req, replicas_status) + ','

                    req.environ['PATH_INFO'] = req_orig_path

                # Report accumulated object status
                accumulated_out = accumulated_out[:-1] + ']'
                return Response(status=HTTP_OK,
                                body=accumulated_out,
                                content_type="text/plain")(env, start_response)

        return self.app(env, start_response)