Exemplo n.º 1
0
class RequestCache(object):

    def __init__(self, app, conf, *args, **kwargs):
        self.app = app
        _servers = conf.get('memcache_servers', '127.0.0.1:11211')
        servers = [s.strip() for s in _servers.split(',') if s.strip()]
        self.memcache = MemcacheRing(servers)

    def __call__(self, env, start_response):

        request = Request(env.copy())
        if request.method != "GET":
            return self.app(env, start_response)

        (version, acc, con, obj) = request.split_path(1, 4, False)
        if not obj:
            return self.app(env, start_response)

        memcache_key = 'RequestCache/%s/%s/%s' % (acc, con, obj)

        cached_content = self.memcache.get(memcache_key)
        if cached_content:
            response = Response(request=request, body=cached_content)
            response.headers['X-RequestCache'] = 'True'
            return response(env, start_response)

        sub_req = wsgi.make_subrequest(env)
        sub_resp = sub_req.get_response(self.app)
        self.memcache.set(memcache_key, sub_resp.body, time=86400.0)

        return self.app(env, start_response)
Exemplo n.º 2
0
def memcached_main(action, options, key=None, value=None):
    memcache_ring = MemcacheRing(
        options['servers'],
        connect_timeout=options['connect_timeout'],
        pool_timeout=options['pool_timeout'],
        tries=options['tries'],
        io_timeout=options['io_timeout'],
        allow_pickle=(options['serialization_format'] == 0),
        allow_unpickle=(options['serialization_format'] <= 1),
        max_conns=options['max_conns'])

    if action == 'set':
        start_time = time.time()
        memcache_ring = get_memcache_ring(options)
        memcache_ring.set(key, value, time=600)
        print('Duration: %s ms' % str((time.time() - start_time) * 1000))
    elif action == 'get':
        start_time = time.time()
        memcache_ring = get_memcache_ring(options)
        value, latency = memcache_get(memcache_ring, key)
        print('Duration: %s ms' % str((time.time() - start_time) * 1000))
        print('Value: %s' % value)
    elif action == 'test_conns':
        test_conns(options)
    elif action == 'test_gets':
        test_gets(memcache_ring, options)
Exemplo n.º 3
0
    def __init__(self, app, conf):
        self.app = app
        self.memcache_servers = conf.get('memcache_servers')
        serialization_format = conf.get('memcache_serialization_support')

        if not self.memcache_servers or serialization_format is None:
            path = os.path.join(conf.get('swift_dir', '/etc/swift'),
                                'memcache.conf')
            memcache_conf = ConfigParser()
            if memcache_conf.read(path):
                if not self.memcache_servers:
                    try:
                        self.memcache_servers = \
                            memcache_conf.get('memcache', 'memcache_servers')
                    except (NoSectionError, NoOptionError):
                        pass
                if serialization_format is None:
                    try:
                        serialization_format = \
                            memcache_conf.get('memcache',
                                              'memcache_serialization_support')
                    except (NoSectionError, NoOptionError):
                        pass

        if not self.memcache_servers:
            self.memcache_servers = '127.0.0.1:11211'
        if serialization_format is None:
            serialization_format = 2

        self.memcache = MemcacheRing(
            [s.strip() for s in self.memcache_servers.split(',') if s.strip()],
            allow_pickle=(serialization_format == 0),
            allow_unpickle=(serialization_format <= 1))
Exemplo n.º 4
0
 def get_cache(self, parsed_args):
     if parsed_args.connection is None:
         parsed_args.connection = self.app.client_manager.sds_conf.get(
             'cache.connection', self.default_connection)
     if parsed_args.connection == self.default_connection:
         self.logger.warn(
             'Using the default connection (%s) is probably '
             'not what you want to do.', self.default_connection)
     _, netloc, _ = parse_connection_string(parsed_args.connection)
     return MemcacheRing(netloc.split(','))
Exemplo n.º 5
0
    def __init__(self, app, conf):
        self.app = app
        self.memcache_servers = conf.get('memcache_servers')
        serialization_format = conf.get('memcache_serialization_support')
        try:
            # Originally, while we documented using memcache_max_connections
            # we only accepted max_connections
            max_conns = int(
                conf.get('memcache_max_connections',
                         conf.get('max_connections', 0)))
        except ValueError:
            max_conns = 0

        if (not self.memcache_servers or serialization_format is None
                or max_conns <= 0):
            path = os.path.join(conf.get('swift_dir', '/etc/swift'),
                                'memcache.conf')
            memcache_conf = ConfigParser()
            if memcache_conf.read(path):
                if not self.memcache_servers:
                    try:
                        self.memcache_servers = \
                            memcache_conf.get('memcache', 'memcache_servers')
                    except (NoSectionError, NoOptionError):
                        pass
                if serialization_format is None:
                    try:
                        serialization_format = \
                            memcache_conf.get('memcache',
                                              'memcache_serialization_support')
                    except (NoSectionError, NoOptionError):
                        pass
                if max_conns <= 0:
                    try:
                        new_max_conns = \
                            memcache_conf.get('memcache',
                                              'memcache_max_connections')
                        max_conns = int(new_max_conns)
                    except (NoSectionError, NoOptionError, ValueError):
                        pass

        if not self.memcache_servers:
            self.memcache_servers = '127.0.0.1:11211'
        if max_conns <= 0:
            max_conns = 2
        if serialization_format is None:
            serialization_format = 2
        else:
            serialization_format = int(serialization_format)

        self.memcache = MemcacheRing(
            [s.strip() for s in self.memcache_servers.split(',') if s.strip()],
            allow_pickle=(serialization_format == 0),
            allow_unpickle=(serialization_format <= 1),
            max_conns=max_conns)
Exemplo n.º 6
0
def get_memcache_ring(options):
    memcache_ring = MemcacheRing(
        options['servers'],
        connect_timeout=options['connect_timeout'],
        pool_timeout=options['pool_timeout'],
        tries=options['tries'],
        io_timeout=options['io_timeout'],
        allow_pickle=(options['serialization_format'] == 0),
        allow_unpickle=(options['serialization_format'] <= 1),
        max_conns=options['max_conns'])
    return memcache_ring
Exemplo n.º 7
0
 def __init__(self, app, conf):
     self.app = app
     self.memcache_servers = conf.get('memcache_servers')
     if not self.memcache_servers:
         path = os.path.join(conf.get('swift_dir', '/etc/swift'),
                             'memcache.conf')
         memcache_conf = ConfigParser()
         if memcache_conf.read(path):
             try:
                 self.memcache_servers = \
                     memcache_conf.get('memcache', 'memcache_servers')
             except (NoSectionError, NoOptionError):
                 pass
     if not self.memcache_servers:
         self.memcache_servers = '127.0.0.1:11211'
     self.memcache = MemcacheRing(
         [s.strip() for s in self.memcache_servers.split(',') if s.strip()])
Exemplo n.º 8
0
 def __init__(self, conf, logger, log_route='log-processor'):
     if isinstance(logger, tuple):
         self.logger = get_logger(*logger, log_route=log_route)
     else:
         self.logger = logger
     self.memcache = MemcacheRing([
         s.strip() for s in conf.get('memcache_servers', '').split(',')
         if s.strip()
     ])
     self.conf = conf
     self._internal_proxy = None
     self.lookback_hours = int(conf.get('lookback_hours', '120'))
     self.lookback_window = int(
         conf.get('lookback_window', self.lookback_hours))
     self.log_processor_account = conf['swift_account']
     self.log_processor_container = conf.get('container_name',
                                             'simple_billing_data')
     self.processed_files_object_name = \
             conf.get('processed_files_object_name',
                     'processed_files.pickle.gz')
Exemplo n.º 9
0
    def __init__(self, app, conf):
        self.app = app
        self.logger = get_logger(conf, log_route='memcache')
        self.memcache_servers = conf.get('memcache_servers')
        serialization_format = conf.get('memcache_serialization_support')
        try:
            # Originally, while we documented using memcache_max_connections
            # we only accepted max_connections
            max_conns = int(
                conf.get('memcache_max_connections',
                         conf.get('max_connections', 0)))
        except ValueError:
            max_conns = 0

        memcache_options = {}
        if (not self.memcache_servers or serialization_format is None
                or max_conns <= 0):
            path = os.path.join(conf.get('swift_dir', '/etc/swift'),
                                'memcache.conf')
            memcache_conf = ConfigParser()
            if memcache_conf.read(path):
                # if memcache.conf exists we'll start with those base options
                try:
                    memcache_options = dict(memcache_conf.items('memcache'))
                except NoSectionError:
                    pass

                if not self.memcache_servers:
                    try:
                        self.memcache_servers = \
                            memcache_conf.get('memcache', 'memcache_servers')
                    except (NoSectionError, NoOptionError):
                        pass
                if serialization_format is None:
                    try:
                        serialization_format = \
                            memcache_conf.get('memcache',
                                              'memcache_serialization_support')
                    except (NoSectionError, NoOptionError):
                        pass
                if max_conns <= 0:
                    try:
                        new_max_conns = \
                            memcache_conf.get('memcache',
                                              'memcache_max_connections')
                        max_conns = int(new_max_conns)
                    except (NoSectionError, NoOptionError, ValueError):
                        pass

        # while memcache.conf options are the base for the memcache
        # middleware, if you set the same option also in the filter
        # section of the proxy config it is more specific.
        memcache_options.update(conf)
        connect_timeout = float(
            memcache_options.get('connect_timeout', CONN_TIMEOUT))
        pool_timeout = float(memcache_options.get('pool_timeout',
                                                  POOL_TIMEOUT))
        tries = int(memcache_options.get('tries', TRY_COUNT))
        io_timeout = float(memcache_options.get('io_timeout', IO_TIMEOUT))
        if config_true_value(memcache_options.get('tls_enabled', 'false')):
            tls_cafile = memcache_options.get('tls_cafile')
            tls_certfile = memcache_options.get('tls_certfile')
            tls_keyfile = memcache_options.get('tls_keyfile')
            self.tls_context = ssl.create_default_context(cafile=tls_cafile)
            if tls_certfile:
                self.tls_context.load_cert_chain(tls_certfile, tls_keyfile)
        else:
            self.tls_context = None
        error_suppression_interval = float(
            memcache_options.get('error_suppression_interval',
                                 ERROR_LIMIT_TIME))
        error_suppression_limit = float(
            memcache_options.get('error_suppression_limit', ERROR_LIMIT_COUNT))

        if not self.memcache_servers:
            self.memcache_servers = '127.0.0.1:11211'
        if max_conns <= 0:
            max_conns = 2
        if serialization_format is None:
            serialization_format = 2
        else:
            serialization_format = int(serialization_format)

        self.memcache = MemcacheRing(
            [s.strip() for s in self.memcache_servers.split(',') if s.strip()],
            connect_timeout=connect_timeout,
            pool_timeout=pool_timeout,
            tries=tries,
            io_timeout=io_timeout,
            allow_pickle=(serialization_format == 0),
            allow_unpickle=(serialization_format <= 1),
            max_conns=max_conns,
            tls_context=self.tls_context,
            logger=self.logger,
            error_limit_count=error_suppression_limit,
            error_limit_time=error_suppression_interval,
            error_limit_duration=error_suppression_interval)
Exemplo n.º 10
0
 def __init__(self, app, conf):
     self.app = app
     self.memcache = MemcacheRing([s.strip() for s in
         conf.get('memcache_servers', '127.0.0.1:11211').split(',')
         if s.strip()])
Exemplo n.º 11
0
    def __init__(self, app, conf):
        self.app = app
        self.memcache_servers = conf.get('memcache_servers')
        serialization_format = conf.get('memcache_serialization_support')
        try:
            # Originally, while we documented using memcache_max_connections
            # we only accepted max_connections
            max_conns = int(
                conf.get('memcache_max_connections',
                         conf.get('max_connections', 0)))
        except ValueError:
            max_conns = 0

        memcache_options = {}
        if (not self.memcache_servers or serialization_format is None
                or max_conns <= 0):
            path = os.path.join(conf.get('swift_dir', '/etc/swift'),
                                'memcache.conf')
            memcache_conf = ConfigParser()
            if memcache_conf.read(path):
                # if memcache.conf exists we'll start with those base options
                try:
                    memcache_options = dict(memcache_conf.items('memcache'))
                except NoSectionError:
                    pass

                if not self.memcache_servers:
                    try:
                        self.memcache_servers = \
                            memcache_conf.get('memcache', 'memcache_servers')
                    except (NoSectionError, NoOptionError):
                        pass
                if serialization_format is None:
                    try:
                        serialization_format = \
                            memcache_conf.get('memcache',
                                              'memcache_serialization_support')
                    except (NoSectionError, NoOptionError):
                        pass
                if max_conns <= 0:
                    try:
                        new_max_conns = \
                            memcache_conf.get('memcache',
                                              'memcache_max_connections')
                        max_conns = int(new_max_conns)
                    except (NoSectionError, NoOptionError, ValueError):
                        pass

        # while memcache.conf options are the base for the memcache
        # middleware, if you set the same option also in the filter
        # section of the proxy config it is more specific.
        memcache_options.update(conf)
        connect_timeout = float(
            memcache_options.get('connect_timeout', CONN_TIMEOUT))
        pool_timeout = float(memcache_options.get('pool_timeout',
                                                  POOL_TIMEOUT))
        tries = int(memcache_options.get('tries', TRY_COUNT))
        io_timeout = float(memcache_options.get('io_timeout', IO_TIMEOUT))

        if not self.memcache_servers:
            self.memcache_servers = '127.0.0.1:11211'
        if max_conns <= 0:
            max_conns = 2
        if serialization_format is None:
            serialization_format = 2
        else:
            serialization_format = int(serialization_format)

        self.memcache = MemcacheRing(
            [s.strip() for s in self.memcache_servers.split(',') if s.strip()],
            connect_timeout=connect_timeout,
            pool_timeout=pool_timeout,
            tries=tries,
            io_timeout=io_timeout,
            allow_pickle=(serialization_format == 0),
            allow_unpickle=(serialization_format <= 1),
            max_conns=max_conns)
Exemplo n.º 12
0
 def __init__(self, app, conf, *args, **kwargs):
     self.app = app
     _servers = conf.get('memcache_servers', '127.0.0.1:11211')
     servers = [s.strip() for s in _servers.split(',') if s.strip()]
     self.memcache = MemcacheRing(servers)