Пример #1
0
def get_bmemcached_client(servers, timeout=None, **kwargs):
    from bmemcached import Client

    if timeout:
        kwargs['socket_timeout'] = timeout

    client = Client(servers, **kwargs)
    client.TooBig = None
    return client
Пример #2
0
def get_bmemcached_client(servers, timeout=None, **kwargs):
    from bmemcached import Client

    if timeout:
        kwargs['socket_timeout'] = timeout

    client = Client(servers, **kwargs)
    client.TooBig = None
    return client
Пример #3
0
class ServicesStatusCache(object):
    """Thin wrapper around cache client to allow for graceful initialization"""
    def __init__(self, servers, services, ttl, binary):
        self.services = services
        self.ttl = ttl
        self._cache = Client(servers, binary=binary)
        self._cache.behaviors = {"no_block": True}

    def initialize(self, service):
        # XXX see if one key is better wrt R/W numbers
        # to do a single memcached call here
        try:
            self._cache.set(_key('service', service, 'on'), True)
            self._cache.set(_key('service', service, 'succ'), 0,
                            time=self.ttl)
            self._cache.set(_key('service', service, 'fail'), 0,
                            time=self.ttl)
            return True
        except WriteError:
            raise StatusWriteError()

    @cache_initialized
    def get(self, key):
        return self._cache.get(key)

    @cache_initialized
    def set(self, key, *args, **kwargs):
        return self._cache.set(key, *args, **kwargs)

    @cache_initialized
    def incr(self, key):
        return self._cache.incr(key)
Пример #4
0
 def mem(self):
     if getattr(self, 'client', None) is None:
         if self.username and self.password:
             self.client = Client(
                 servers=self.servers,
                 username=self.username,
                 password=self.password,
                 binary=True
             )
         else:
             self.client = Client(self.servers)
     return self.client
Пример #5
0
def memcache_client(servers):
    """Context-manager for creating connections to memcache.

    Unfortunately pylibmc doesn't seem to play nicely with os.fork(),
    so we can't create a single connection and use it for all queries.
    Instead we must create one when needed and tear it down when done.
    """
    client = Client(servers, behaviors={"cas": 1})
    try:
        yield client
    finally:
        client.disconnect_all()
Пример #6
0
def memcache_client(servers):
    """Context-manager for creating connections to memcache.

    Unfortunately pylibmc doesn't seem to play nicely with os.fork(),
    so we can't create a single connection and use it for all queries.
    Instead we must create one when needed and tear it down when done.
    """
    client = Client(servers, behaviors={"cas": 1})
    try:
        yield client
    finally:
        client.disconnect_all()
Пример #7
0
class MemcacheCache(api.APICache):
	def __init__(self, config):
		super(MemcacheCache, self).__init__()
		self.mc = Client(config['MEMCACHE'])

	def get(self, key):
		return self.mc.get(key)
		
	def set(self, key, value, duration):
		if duration < 0:
			duration = time.time() + duration
		self.mc.set(key, value, time=duration)
Пример #8
0
def get_pylibmc_client(servers, timeout=None, binary=True, **kwargs):
    from pylibmc import Client

    try:
        from pylibmc import TooBig
    except ImportError:
        from pylibmc import Error, ServerError
        TooBig = (Error, ServerError)

    if timeout:
        kwargs['behaviors'] = {'connect_timeout': timeout}

    client = Client(servers, binary=binary, **kwargs)
    client.TooBig = TooBig
    return client
Пример #9
0
def get_pylibmc_client(servers, timeout=None, binary=True, **kwargs):
    from pylibmc import Client

    try:
        from pylibmc import TooBig
    except ImportError:
        from pylibmc import Error, ServerError
        TooBig = (Error, ServerError)

    if timeout:
        kwargs['behaviors'] = {'connect_timeout': timeout}

    client = Client(servers, binary=binary, **kwargs)
    client.TooBig = TooBig
    return client
Пример #10
0
def get_memcached_client(config=None):
    try:
        from pylibmc import Client
    except ImportError:
        # try to use an alternative client
        from memcache import Client as BaseClient

        class Client(BaseClient):
            def add_multi(self,
                          mapping,
                          time=0,
                          key_prefix='',
                          min_compress_len=0,
                          noreply=False):
                """
                Behaviour of set_multi is some different but there is no other alternative
                """
                return self.set_multi(mapping, time, key_prefix,
                                      min_compress_len, noreply)

            def delete(self, key, time=None, noreply=False):
                """
                The logic of delete method is some different in memcache and pylibmc client
                pylibmc client returns True only if key exists in cache
                memcache client always returns True
                This patch fixes the situation
                """
                return self._deletetouch([b'DELETED'], "delete", key, time,
                                         noreply)

    memcached_params = dict({'servers': ['localhost']}, **config or {})
    return Client(**memcached_params)
Пример #11
0
    def __init__(self, urls, controllers, config=None,
                 auth_class=Authentication):
        super(StorageServerApp, self).__init__(urls, controllers, config,
                                               auth_class)
        self.config = config

        # Collecting the host-specific config and building connectors.
        self.storages = {'default': get_storage(config)}
        hostnames = set()
        host_token = 'host:'
        for cfgkey in config:
            if cfgkey.startswith(host_token):
                # Get the hostname from the config key.  This assumes
                # that host-specific keys have two trailing components
                # that specify the setting to override.
                # E.g: "host:localhost.storage.sqluri" => "localhost"
                hostname = cfgkey[len(host_token):].rsplit(".", 2)[0]
                hostnames.add(hostname)
        for hostname in hostnames:
            host_cfg = self._host_specific(hostname, config)
            self.storages[hostname] = get_storage(host_cfg)

        # If we need to check node status, then we need to
        # obtain a memcache client object.
        self.cache = None
        self.check_node_status = \
                self.config.get('storage.check_node_status', False)
        if self.check_node_status:
            if Client is None:
                raise ValueError('The "check_node_status" option '
                                 'needs a memcached server')
            servers = self.config.get('storage.cache_servers',
                                      '127.0.0.1:11211')
            self.cache = Client(servers.split(','))
Пример #12
0
 def __init__(self, servers):
     """Initialize memcached.
     @param servers: an array of servers. Servers can be passed in two forms:
         1. Strings of the form host:port (implies a default weight of 1).
         2. Tuples of the form (host:port, weight) (weight as integer)
     """
     self._client = Client(servers)
     self.timeout = 0
Пример #13
0
 def __init__(self,
              *,
              namespace="dramatiq-results",
              pool=None,
              pool_size=8,
              **parameters):
     self.namespace = namespace
     self.pool = pool or ClientPool(Client(**parameters), pool_size)
 def __init__(self, *args, **kw):
     self._client = Client(*args, **kw)
     self.pool = ThreadMappedPool(self._client)
     # using a locker to avoid race conditions
     # when several clients for the same user
     # get/set the cached data
     self._locker = threading.RLock()
     subscribe(REQUEST_ENDS, self._cleanup_pool)
Пример #15
0
def getmc():
    try:
        from pylibmc import Client
        from config import cacheServers
        adv_setting = {"cas": True, "tcp_nodelay": True, "ketama": True}
        return Client(cacheServers, binary=True, behaviors=adv_setting)
    except Exception as e:
        print e
        print "import pylibmc failed, perhaps your os is not supported"
Пример #16
0
    def __init__(self, urls, controllers, config=None,
                 auth_class=WhoAuthentication):
        super(StorageServerApp, self).__init__(urls, controllers, config,
                                               auth_class)
        self.config = config

        # Collecting the host-specific config and building connectors.
        self.storages = {'default': get_storage(config)}
        hostnames = set()
        host_token = 'host:'
        for cfgkey in config:
            if cfgkey.startswith(host_token):
                # Get the hostname from the config key.  This assumes
                # that host-specific keys have two trailing components
                # that specify the setting to override.
                # E.g: "host:localhost.storage.sqluri" => "localhost"
                hostname = cfgkey[len(host_token):].rsplit(".", 2)[0]
                hostnames.add(hostname)
        for hostname in hostnames:
            host_cfg = self._host_specific(hostname, config)
            self.storages[hostname] = get_storage(host_cfg)

        # If we need to check node status, then we need to
        # obtain a memcache client object.
        self.cache = None
        self.check_node_status = \
                self.config.get('storage.check_node_status', False)
        if self.check_node_status:
            if Client is None:
                raise ValueError('The "check_node_status" option '
                                 'needs a memcached server')
            servers = self.config.get('storage.cache_servers',
                                      '127.0.0.1:11211')
            self.cache = Client(servers.split(','))

        # Config knobs for managing staged rollout of EOL headers.
        self.eol_header_value = json.dumps({
            'code': self.config.get('storage.eol_header_code', 'soft-eol'),
            'message': self.config.get('storage.eol_header_message', ''),
            'url': self.config.get('storage.eol_header_url', ''),
        })
        self.eol_general_release = \
                self.config.get('storage.eol_general_release', False)
        self.eol_rollout_percent = \
                int(self.config.get('storage.eol_rollout_percent', 0))
        self.eol_rollout_percent_hwm = \
                int(self.config.get('storage.eol_rollout_percent_hwm', 0))
        if not 0 <= self.eol_rollout_percent <= 100:
            raise ValueError('storage.eol_rollout_percent should be ' \
                             'between 0 and 100')
        if not 0 <= self.eol_rollout_percent_hwm <= 100:
            raise ValueError('storage.eol_rollout_percent_hwm should be ' \
                             'between 0 and 100')
        if self.eol_rollout_percent_hwm > 0:
            if not self.eol_rollout_percent_hwm >= self.eol_rollout_percent:
                raise ValueError('storage.eol_rollout_percent_hwm should be ' \
                                 'greater than storage.eol_rollout_percent')
Пример #17
0
def cache(localconfig=None):
    "cache object"
    if localconfig:
        server = localconfig.get('baruwa.memcached.host', '127.0.0.1')
    else:
        server = config.get('baruwa.memcached.host', '127.0.0.1')
    beh = {"tcp_nodelay": True, "ketama": True}
    conn = Client([server], binary=True, behaviors=beh)
    return conn
Пример #18
0
 def __init__(self,
              *,
              namespace="dramatiq-results",
              encoder=None,
              pool=None,
              pool_size=8,
              **parameters):
     super().__init__(namespace=namespace, encoder=encoder)
     self.pool = pool or ClientPool(Client(**parameters), pool_size)
Пример #19
0
 def __init__(self, **options):
     self.servers = [
         server.strip() for server in options['servers'].split(',')
     ]
     self.prefix = options['prefix']
     self._client = Client(self.servers)
     self.pool = ThreadMappedPool(self._client)
     # using a locker to avoid race conditions
     # when several clients for the same user
     # get/set the cached data
     self._locker = threading.RLock()
Пример #20
0
def getmc():
    """
    @return: Memcached ClientPool
    """
    try:
        from pylibmc import Client, ClientPool
        from config import cacheServers, MC_POOL_SIZE
        adv_setting = {"cas": True, "tcp_nodelay": True, "ketama": True}
        mc = Client(cacheServers, binary=True, behaviors=adv_setting)
        return ClientPool(mc, MC_POOL_SIZE)
    except ImportError:
        return ClientPoolEmptyOnWIndows()
Пример #21
0
def _setup_cache(app):
    """
    If a test is being run or we don't want cache, NullCache will be initialized just as a dummy.
    If running locally without the 'DISABLE_CACHE' env variable and without a memcached instance running,
    MemcachedCache and it's underlying pylibmc will give no warning on connection, but will throw
    exceptions when trying to work with the cache. A few connection retires will be made in that
    scenario, and eventually the cache will be replaced with a NullCache. Binary communications must
    be used for SASL.
    """

    # initialize the retry count if it's our first time here
    if not hasattr(app, 'cache_retry'):
        app.cache_retry = 0

    # Setup cache
    if app.config['TESTING'] or os.environ.get('DISABLE_CACHE',
                                               None) is not None:
        app.cache = NullCache()
        app.logger.debug('Cache initialized as NullCache')
    else:
        MEMCACHED_SERVERS = os.environ.get('MEMCACHEDCLOUD_SERVERS',
                                           '127.0.0.1:11211')

        try:
            memcached_client = Client(
                servers=MEMCACHED_SERVERS.split(','),
                username=os.environ.get('MEMCACHEDCLOUD_USERNAME'),
                password=os.environ.get('MEMCACHEDCLOUD_PASSWORD'),
                binary=True)
            app.cache = MemcachedCache(memcached_client)
            app.logger.debug(
                'Cache initialized as MemcachedCache with servers: %s',
                MEMCACHED_SERVERS)
        except Exception as e:
            # very unlikely to have an exception here. pylibmc mostly throws when trying to communicate, not connect
            app.logger.error('Error initializing MemcachedCache: %s', e)
            app.logger.error('Initializing cache as NullCache. Fix ASAP!')
            app.cache = NullCache()
Пример #22
0
    def __init__(self, *, pool=None, pool_size=8, **parameters):
        behaviors = parameters.setdefault("behaviors", {})
        behaviors["cas"] = True

        self.pool = pool or ClientPool(Client(**parameters), pool_size)
Пример #23
0
app = Flask(__name__)
servers = os.environ.get('MEMCACHIER_SERVERS', '').split(',')
user = os.environ.get('MEMCACHIER_USERNAME', '')
passw = os.environ.get('MEMCACHIER_PASSWORD', '')
# Client config obtained from https://devcenter.heroku.com/articles/memcachier#python
mc = Client(
    servers,
    binary=True,
    username=user,
    password=passw,
    behaviors={
        # Faster IO
        "tcp_nodelay": True,

        # Keep connection alive
        'tcp_keepalive': True,

        # Timeout for set/get requests
        'connect_timeout': 2000,  # ms
        'send_timeout': 750 * 1000,  # us
        'receive_timeout': 750 * 1000,  # us
        '_poll_timeout': 2000,  # ms

        # Better failover
        'ketama': True,
        'remove_failed': 1,
        'retry_timeout': 2,
        'dead_timeout': 30,
    })

from app import routes
Пример #24
0
def create_app():
    """Instanciate app."""
    app = Flask(__name__)

    if os.path.exists(app.config.root_path + '/config.cfg') is False:
        print("copy config.default.cfg to config.cfg and add your settings")
        app.config.from_pyfile(app.config.root_path + '/config.default.cfg')
    else:
        app.config.from_pyfile(app.config.root_path + '/config.cfg')

    babel = Babel(app)
    Compress(app)

    @babel.localeselector
    def get_locale():
        """Get correct language from url."""
        locale = request.path[1:].split('/', 1)[0]
        if locale in ['sv', 'en']:
            return locale
        else:
            locale = 'sv'
            for lang in list(request.accept_languages.values()):
                if lang[:2] in ['sv', 'en']:
                    locale = lang[:2]
                    break

            g.locale = locale
            return locale

    client = Client(app.config['MEMCACHED'])

    @app.before_request
    def func():
        g.babel = Babel
        g.language = get_locale()
        g.config = app.config
        g.mc_pool = ClientPool(client, app.config['POOL_SIZE'])

    @app.context_processor
    def inject_custom():
        d = {
            'lurl_for':
            lambda ep, **kwargs: url_for(ep + '_' + g.language, **kwargs)
        }
        return d

    @app.template_filter('deescape')
    def deescape_filter(s):
        html_parser = html.parser.HTMLParser()
        return html_parser.unescape(s)

    @app.template_filter('cclink')
    def cclink_filter(s):
        return re.sub(
            r'(CC-BY\S*)',
            '<a href="https://creativecommons.org/licenses/" target="_blank">\\1</a>',
            s)

    from . import helpers

    app.jinja_env.globals.update(get_life_range=helpers.get_life_range)
    app.jinja_env.globals.update(make_namelist=helpers.make_namelist)
    app.jinja_env.globals.update(make_datelist=helpers.make_datelist)
    app.jinja_env.globals.update(
        make_simplenamelist=helpers.make_simplenamelist)
    app.jinja_env.globals.update(make_placelist=helpers.make_placelist)
    app.jinja_env.globals.update(make_placenames=helpers.make_placenames)
    app.jinja_env.globals.update(
        make_alphabetical_bucket=helpers.make_alphabetical_bucket)
    app.jinja_env.globals.update(get_date=helpers.get_date)
    app.jinja_env.globals.update(join_name=helpers.join_name)
    app.jinja_env.globals.update(swedish_translator=helpers.swedish_translator)
    app.jinja_env.globals.update(sorted=sorted)
    app.jinja_env.globals.update(len=len)
    app.jinja_env.globals.update(get_lang_text=helpers.get_lang_text)
    app.jinja_env.globals.update(get_shorttext=helpers.get_shorttext)
    app.jinja_env.globals.update(get_org_name=helpers.get_org_name)
    app.jinja_env.globals.update(rewrite_von=helpers.rewrite_von)
    app.jinja_env.globals.update(lowersorted=helpers.lowersorted)
    app.jinja_env.globals.update(get_current_date=helpers.get_current_date)
    app.jinja_env.globals.update(karp_fe_url=helpers.karp_fe_url)

    from . import views
    app.register_blueprint(views.bp)
    app.register_error_handler(Exception, views.page_not_found)

    app.wsgi_app = flask_reverse_proxy.ReverseProxied(app.wsgi_app)
    return app
Пример #25
0
	def __init__(self, config):
		super(MemcacheCache, self).__init__()
		self.mc = Client(config['MEMCACHE'])
Пример #26
0
class StorageServerApp(SyncServerApp):
    """Storage application"""
    def __init__(self, urls, controllers, config=None,
                 auth_class=WhoAuthentication):
        super(StorageServerApp, self).__init__(urls, controllers, config,
                                               auth_class)
        self.config = config

        # Collecting the host-specific config and building connectors.
        self.storages = {'default': get_storage(config)}
        hostnames = set()
        host_token = 'host:'
        for cfgkey in config:
            if cfgkey.startswith(host_token):
                # Get the hostname from the config key.  This assumes
                # that host-specific keys have two trailing components
                # that specify the setting to override.
                # E.g: "host:localhost.storage.sqluri" => "localhost"
                hostname = cfgkey[len(host_token):].rsplit(".", 2)[0]
                hostnames.add(hostname)
        for hostname in hostnames:
            host_cfg = self._host_specific(hostname, config)
            self.storages[hostname] = get_storage(host_cfg)

        # If we need to check node status, then we need to
        # obtain a memcache client object.
        self.cache = None
        self.check_node_status = \
                self.config.get('storage.check_node_status', False)
        if self.check_node_status:
            if Client is None:
                raise ValueError('The "check_node_status" option '
                                 'needs a memcached server')
            servers = self.config.get('storage.cache_servers',
                                      '127.0.0.1:11211')
            self.cache = Client(servers.split(','))

        # Config knobs for managing staged rollout of EOL headers.
        self.eol_header_value = json.dumps({
            'code': self.config.get('storage.eol_header_code', 'soft-eol'),
            'message': self.config.get('storage.eol_header_message', ''),
            'url': self.config.get('storage.eol_header_url', ''),
        })
        self.eol_general_release = \
                self.config.get('storage.eol_general_release', False)
        self.eol_rollout_percent = \
                int(self.config.get('storage.eol_rollout_percent', 0))
        self.eol_rollout_percent_hwm = \
                int(self.config.get('storage.eol_rollout_percent_hwm', 0))
        if not 0 <= self.eol_rollout_percent <= 100:
            raise ValueError('storage.eol_rollout_percent should be ' \
                             'between 0 and 100')
        if not 0 <= self.eol_rollout_percent_hwm <= 100:
            raise ValueError('storage.eol_rollout_percent_hwm should be ' \
                             'between 0 and 100')
        if self.eol_rollout_percent_hwm > 0:
            if not self.eol_rollout_percent_hwm >= self.eol_rollout_percent:
                raise ValueError('storage.eol_rollout_percent_hwm should be ' \
                                 'greater than storage.eol_rollout_percent')

    def get_storage(self, request):
        host = request.host
        if host not in self.storages:
            host = 'default'
        return self.storages[host]

    def _before_call(self, request):
        headers = {}

        # If configured to do so, this function can check memcache for
        # the status of the target node and possibly avoid calling out
        # to the storage backend.  It looks for a memcache key named
        # "status:<hostname>" with one of the following values:
        #
        #    "down":   the node is explicitly marked as down
        #    "draining":   the node is being decommissioned
        #    "unhealthy":  the node has not responded to health checks
        #    "backoff" or "backoff:NN":  the node is under heavy load and
        #                                clients should back off for a while.

        if self.check_node_status:

            # Helper function to create a HTTPServiceUnavailable response.
            # This takes care of some fiddly details in the API.
            def resp_service_unavailable(msg):
                body = "server issue: " + msg
                headers["Retry-After"] = str(self.retry_after)
                headers["X-Weave-Backoff"] = str(self.retry_after)
                raise HTTPServiceUnavailable(headers=headers,
                                             body_template=body)

            # Get the node name from host header,
            # and check that it's one we know about.
            node = request.host
            if not node:
                msg = "host header not received from client"
                raise resp_service_unavailable(msg)

            if node not in self.storages:
                msg = "database lookup failed"
                raise resp_service_unavailable(msg)

            status = self.cache.get('status:%s' % request.host)
            if status is not None:

                # If it's marked as draining then send a 503 response.
                # XXX TODO: consider sending a 401 to trigger migration?
                if status == "draining":
                    msg = "node reassignment"
                    raise resp_service_unavailable(msg)

                # If it's marked as being down then send a 503 response.
                if status == "down":
                    msg = "database marked as down"
                    raise resp_service_unavailable(msg)

                # If it's marked as being unhealthy then send a 503 response.
                if status == "unhealthy":
                    msg = "database is not healthy"
                    raise resp_service_unavailable(msg)

                # If it's marked for backoff, proceed with the request
                # but set appropriate headers on the response.
                if status == "backoff" or status.startswith("backoff:"):
                    try:
                        backoff = status.split(":", 1)[1]
                    except IndexError:
                        backoff = str(self.retry_after)
                    headers["X-Weave-Backoff"] = backoff

        return headers

    def _debug_server(self, request):
        res = []
        storage = self.get_storage(request)
        res.append('- backend: %s' % storage.get_name())
        if storage.get_name() in ('memcached',):
            cache_servers = ['%s:%d' % (server.ip, server.port)
                             for server in storage.cache.servers]
            res.append('- memcached servers: %s</li>' %
                       ', '.join(cache_servers))

        if storage.get_name() in ('sql', 'memcached'):
            res.append('- sqluri: %s' % storage.sqluri)
        return res

    def _dispatch_request_with_match(self, request, match):
        # Manage staged rollout of end-of-life headers.
        # We need to do it in this method rather than _before_call(),
        # because the logic needs access to the numeric userid.

        try:
            userid = int(request.user['userid'])
        except (AttributeError, KeyError):
            # In e.g. self-hosted environments, we may not have a userid
            # on all requests.  Just process them normally.
            supercls = super(StorageServerApp, self)
            return supercls._dispatch_request_with_match(request, match)

        # We're going to test the EOL headers while the corresponding
        # client code is still riding the trains.  Until general release, it's
        # only safe to show them to sufficiently-recent desktop devices.

        if self.eol_general_release:
            is_eligible_user_agent = True
        else:
            regex = USER_AGENT_WITH_EOL_SUPPORT
            user_agent = request.headers.get("User-Agent", "")
            is_eligible_user_agent = (regex.match(user_agent) is not None)

        # Show the EOL headers to the configured percentage of users,
        # based on the low-order bits of their userid.  This is a simple
        # way to ensure that users will consistently see headers on or off.

        should_see_eol_headers = (is_eligible_user_agent and
                                  userid % 100 < self.eol_rollout_percent)

        # Before general release, we only want to show the headers to users
        # with a single device connected.  Sadly, the only way to check this
        # is by doing a massive layering violation and peeking at the DB.

        if should_see_eol_headers and not self.eol_general_release:
            storage = self.get_storage(request)
            client_records = storage.get_items(userid, "clients", limit=2)
            if len(client_records) > 1:
                should_see_eol_headers = False

        # If we ever make the eol_rollout_percent smaller, it will mean that
        # some users stop seeing EOL headers where previously they were seeing
        # them.  This could interfere with in-progress migrations, so we need
        # to check whether those users have already started the process, and
        # keep sending headers if so.

        if not should_see_eol_headers:
            if self.eol_rollout_percent_hwm > self.eol_rollout_percent:
                if userid % 100 < self.eol_rollout_percent_hwm:
                    # Check in the DB whether this user has started migrating.
                    # It's a massive layering violation, but gets the job done!
                    storage = self.get_storage(request)
                    if storage.get_item(userid, "meta", "fxa_credentials"):
                        should_see_eol_headers = True

        # Log some metrics to help track progress and performance of migration:
        #  * count of requests for which we considered sending the header
        #  * count of requests for which we actually sent the header
        #  * count of reads and writes to the migration sentinel
        #  * count of client record cleanups by users who received a header

        if is_eligible_user_agent:
            self.logger.incr(CTR_EOL_HEADER_CONSIDERED)
        if should_see_eol_headers:
            self.logger.incr(CTR_EOL_HEADER_SENT)
        if request.method == "GET":
            if match.get("collection") == "meta":
                if match.get("item") == "fxa_credentials":
                    self.logger.incr(CTR_EOL_SENTINEL_READ)
        elif request.method in ("PUT", "POST"):
            if match.get("collection") == "meta":
                if match.get("item") == "fxa_credentials":
                    self.logger.incr(CTR_EOL_SENTINEL_WRITE)
        elif request.method == "DELETE":
            if should_see_eol_headers:
                self.logger.incr(CTR_EOL_CLIENT_DELETE)

        # Continue with request dispatch, ensuring that we set the appropriate
        # headers on both success and failure responses.

        supercls = super(StorageServerApp, self)
        try:
            response = supercls._dispatch_request_with_match(request, match)
        except HTTPException, response:
            if should_see_eol_headers:
                response.headers['X-Weave-Alert'] = self.eol_header_value
            raise
        else:
Пример #27
0
 def __init__(self, servers, services, ttl, binary):
     self.services = services
     self.ttl = ttl
     self._cache = Client(servers, binary=binary)
     self._cache.behaviors = {"no_block": True}
Пример #28
0
from urllib2 import Request, urlopen
import HTMLParser
import helpers

app = Flask(__name__)

if os.path.exists(app.config.root_path + '/config.cfg') is False:
    print "copy config.default.cfg to config.cfg and add your settings"
    app.config.from_pyfile(app.config.root_path + '/config.default.cfg')
else:
    app.config.from_pyfile(app.config.root_path + '/config.cfg')

babel = Babel(app)
Compress(app)

client = Client(app.config['MEMCACHED'])
mc_pool = ClientPool(client, app.config['POOL_SIZE'])


def cache_name(pagename, lang=''):
    if not lang:
        lang = 'sv' if 'sv' in request.url_rule.rule else 'en'
    return '%s_%s' % (pagename, lang)


def check_cache(page, lang=''):
    # If the cache should not be used, return None
    if app.config['TEST']:
        return None
    try:
        with mc_pool.reserve() as client:
Пример #29
0
    def __init__(self, *, pool_size=8, **parameters):
        behaviors = parameters["behaviors"] = parameters.get("behaviors", {})
        behaviors["cas"] = True

        self.client = client = Client(**parameters)
        self.pool = ClientPool(client, pool_size)
Пример #30
0
class StorageServerApp(SyncServerApp):
    """Storage application"""
    def __init__(self, urls, controllers, config=None,
                 auth_class=Authentication):
        super(StorageServerApp, self).__init__(urls, controllers, config,
                                               auth_class)
        self.config = config

        # Collecting the host-specific config and building connectors.
        self.storages = {'default': get_storage(config)}
        hostnames = set()
        host_token = 'host:'
        for cfgkey in config:
            if cfgkey.startswith(host_token):
                # Get the hostname from the config key.  This assumes
                # that host-specific keys have two trailing components
                # that specify the setting to override.
                # E.g: "host:localhost.storage.sqluri" => "localhost"
                hostname = cfgkey[len(host_token):].rsplit(".", 2)[0]
                hostnames.add(hostname)
        for hostname in hostnames:
            host_cfg = self._host_specific(hostname, config)
            self.storages[hostname] = get_storage(host_cfg)

        # If we need to check node status, then we need to
        # obtain a memcache client object.
        self.cache = None
        self.check_node_status = \
                self.config.get('storage.check_node_status', False)
        if self.check_node_status:
            if Client is None:
                raise ValueError('The "check_node_status" option '
                                 'needs a memcached server')
            servers = self.config.get('storage.cache_servers',
                                      '127.0.0.1:11211')
            self.cache = Client(servers.split(','))

    def get_storage(self, request):
        host = request.host
        if host not in self.storages:
            host = 'default'
        return self.storages[host]

    def _before_call(self, request):
        headers = {}

        # If configured to do so, this function can check memcache for
        # the status of the target node and possibly avoid calling out
        # to the storage backend.  It looks for a memcache key named
        # "status:<hostname>" with one of the following values:
        #
        #    "down":   the node is explicitly marked as down
        #    "draining":   the node is being decommissioned
        #    "unhealthy":  the node has not responded to health checks
        #    "backoff" or "backoff:NN":  the node is under heavy load and
        #                                clients should back off for a while.

        if self.check_node_status:

            # Helper function to create a HTTPServiceUnavailable response.
            # This takes care of some fiddly details in the API.
            def resp_service_unavailable(msg):
                body = "server issue: " + msg
                headers["Retry-After"] = str(self.retry_after)
                headers["X-Weave-Backoff"] = str(self.retry_after)
                raise HTTPServiceUnavailable(headers=headers,
                                             body_template=body)

            # Get the node name from host header,
            # and check that it's one we know about.
            node = request.host
            if not node:
                msg = "host header not received from client"
                raise resp_service_unavailable(msg)

            if node not in self.storages:
                msg = "database lookup failed"
                raise resp_service_unavailable(msg)

            status = self.cache.get('status:%s' % request.host)
            if status is not None:

                # If it's marked as draining then send a 503 response.
                # XXX TODO: consider sending a 401 to trigger migration?
                if status == "draining":
                    msg = "node reassignment"
                    raise resp_service_unavailable(msg)

                # If it's marked as being down then send a 503 response.
                if status == "down":
                    msg = "database marked as down"
                    raise resp_service_unavailable(msg)

                # If it's marked as being unhealthy then send a 503 response.
                if status == "unhealthy":
                    msg = "database is not healthy"
                    raise resp_service_unavailable(msg)

                # If it's marked for backoff, proceed with the request
                # but set appropriate headers on the response.
                if status == "backoff" or status.startswith("backoff:"):
                    try:
                        backoff = status.split(":", 1)[1]
                    except IndexError:
                        backoff = str(self.retry_after)
                    headers["X-Weave-Backoff"] = backoff

        return headers

    def _debug_server(self, request):
        res = []
        storage = self.get_storage(request)
        res.append('- backend: %s' % storage.get_name())
        if storage.get_name() in ('memcached',):
            cache_servers = ['%s:%d' % (server.ip, server.port)
                             for server in storage.cache.servers]
            res.append('- memcached servers: %s</li>' %
                       ', '.join(cache_servers))

        if storage.get_name() in ('sql', 'memcached'):
            res.append('- sqluri: %s' % storage.sqluri)
        return res
Пример #31
0
with open(os.path.join(APP_STATIC, 'post_data_stable.xml')) as f:
    POST_DATA_STABLE = f.read().replace('\n', '')

with open(os.path.join(APP_STATIC, 'post_data_beta.xml')) as f:
    POST_DATA_BETA = f.read().replace('\n', '')

with open(os.path.join(APP_STATIC, 'post_data_dev.xml')) as f:
    POST_DATA_DEV = f.read().replace('\n', '')

post_data = OrderedDict([('stable', POST_DATA_STABLE),
                         ('beta', POST_DATA_BETA),
                         ('dev', POST_DATA_DEV)])

if MEMCACHE_SERVERS:
    client = Client([MEMCACHE_SERVERS], behaviors={"tcp_nodelay": True},
                    binary=True, username=MEMCACHE_USERNAME, password=MEMCACHE_PASSWORD)
else:
    client = Client(['127.0.0.1:11211'])
cache = MemcachedCache(client, default_timeout=60)


def cached(timeout=60):
    def decorator(f):
        @wraps(f)
        def decorated_function(*args, **kwargs):
            key = args[0]
            rv = cache.get(key)
            if rv is not None:
                print 'hit\t\t', key
                return rv
            print 'miss\t\t', key
Пример #32
0
class Memcached(object):
    
    implements(IMemcachedProvider)
    
    def __init__(self, servers):
        """Initialize memcached.
        @param servers: an array of servers. Servers can be passed in two forms:
            1. Strings of the form host:port (implies a default weight of 1).
            2. Tuples of the form (host:port, weight) (weight as integer)
        """
        self._client = Client(servers)
        self.timeout = 0
          
    def reset(self):
        self._client.flush_all()
    
    def size(self):
        bytes = 0
        stats = self._client.get_stats()
        for name, stat in stats:
            bytes+=int(stat['bytes'])
        return bytes

    def keys(self):
        raise MemcachedException, \
              "It's not possible to fetch keys from memcached"
                
    def values(self):
        raise MemcachedException, \
              "It's not possible to fetch values from memcached"
    
    def get(self, key, default=None):
        value = self._client.get(key)
        if value is not None:
            return value 
        return default
    
    def __getitem__(self, key):
        return self._client.get(key)
    
    def __setitem__(self, key, object):
        if PYLIBMC:
            self._client.set(key, object, time=self.timeout,
                             min_compress_len=1024000,
                             compress_level=zlib.Z_BEST_SPEED)
        else:
            self._client.set(key, object, time=self.timeout)

    def __delitem__(self, key):
        self._client.delete(key)