def get_bmemcached_client(servers, timeout=None, **kwargs): from bmemcached import Client if timeout: kwargs['socket_timeout'] = timeout client = Client(servers, **kwargs) client.TooBig = None return client
class ServicesStatusCache(object): """Thin wrapper around cache client to allow for graceful initialization""" def __init__(self, servers, services, ttl, binary): self.services = services self.ttl = ttl self._cache = Client(servers, binary=binary) self._cache.behaviors = {"no_block": True} def initialize(self, service): # XXX see if one key is better wrt R/W numbers # to do a single memcached call here try: self._cache.set(_key('service', service, 'on'), True) self._cache.set(_key('service', service, 'succ'), 0, time=self.ttl) self._cache.set(_key('service', service, 'fail'), 0, time=self.ttl) return True except WriteError: raise StatusWriteError() @cache_initialized def get(self, key): return self._cache.get(key) @cache_initialized def set(self, key, *args, **kwargs): return self._cache.set(key, *args, **kwargs) @cache_initialized def incr(self, key): return self._cache.incr(key)
def mem(self): if getattr(self, 'client', None) is None: if self.username and self.password: self.client = Client( servers=self.servers, username=self.username, password=self.password, binary=True ) else: self.client = Client(self.servers) return self.client
def memcache_client(servers): """Context-manager for creating connections to memcache. Unfortunately pylibmc doesn't seem to play nicely with os.fork(), so we can't create a single connection and use it for all queries. Instead we must create one when needed and tear it down when done. """ client = Client(servers, behaviors={"cas": 1}) try: yield client finally: client.disconnect_all()
class MemcacheCache(api.APICache): def __init__(self, config): super(MemcacheCache, self).__init__() self.mc = Client(config['MEMCACHE']) def get(self, key): return self.mc.get(key) def set(self, key, value, duration): if duration < 0: duration = time.time() + duration self.mc.set(key, value, time=duration)
def get_pylibmc_client(servers, timeout=None, binary=True, **kwargs): from pylibmc import Client try: from pylibmc import TooBig except ImportError: from pylibmc import Error, ServerError TooBig = (Error, ServerError) if timeout: kwargs['behaviors'] = {'connect_timeout': timeout} client = Client(servers, binary=binary, **kwargs) client.TooBig = TooBig return client
def get_memcached_client(config=None): try: from pylibmc import Client except ImportError: # try to use an alternative client from memcache import Client as BaseClient class Client(BaseClient): def add_multi(self, mapping, time=0, key_prefix='', min_compress_len=0, noreply=False): """ Behaviour of set_multi is some different but there is no other alternative """ return self.set_multi(mapping, time, key_prefix, min_compress_len, noreply) def delete(self, key, time=None, noreply=False): """ The logic of delete method is some different in memcache and pylibmc client pylibmc client returns True only if key exists in cache memcache client always returns True This patch fixes the situation """ return self._deletetouch([b'DELETED'], "delete", key, time, noreply) memcached_params = dict({'servers': ['localhost']}, **config or {}) return Client(**memcached_params)
def __init__(self, urls, controllers, config=None, auth_class=Authentication): super(StorageServerApp, self).__init__(urls, controllers, config, auth_class) self.config = config # Collecting the host-specific config and building connectors. self.storages = {'default': get_storage(config)} hostnames = set() host_token = 'host:' for cfgkey in config: if cfgkey.startswith(host_token): # Get the hostname from the config key. This assumes # that host-specific keys have two trailing components # that specify the setting to override. # E.g: "host:localhost.storage.sqluri" => "localhost" hostname = cfgkey[len(host_token):].rsplit(".", 2)[0] hostnames.add(hostname) for hostname in hostnames: host_cfg = self._host_specific(hostname, config) self.storages[hostname] = get_storage(host_cfg) # If we need to check node status, then we need to # obtain a memcache client object. self.cache = None self.check_node_status = \ self.config.get('storage.check_node_status', False) if self.check_node_status: if Client is None: raise ValueError('The "check_node_status" option ' 'needs a memcached server') servers = self.config.get('storage.cache_servers', '127.0.0.1:11211') self.cache = Client(servers.split(','))
def __init__(self, servers): """Initialize memcached. @param servers: an array of servers. Servers can be passed in two forms: 1. Strings of the form host:port (implies a default weight of 1). 2. Tuples of the form (host:port, weight) (weight as integer) """ self._client = Client(servers) self.timeout = 0
def __init__(self, *, namespace="dramatiq-results", pool=None, pool_size=8, **parameters): self.namespace = namespace self.pool = pool or ClientPool(Client(**parameters), pool_size)
def __init__(self, *args, **kw): self._client = Client(*args, **kw) self.pool = ThreadMappedPool(self._client) # using a locker to avoid race conditions # when several clients for the same user # get/set the cached data self._locker = threading.RLock() subscribe(REQUEST_ENDS, self._cleanup_pool)
def getmc(): try: from pylibmc import Client from config import cacheServers adv_setting = {"cas": True, "tcp_nodelay": True, "ketama": True} return Client(cacheServers, binary=True, behaviors=adv_setting) except Exception as e: print e print "import pylibmc failed, perhaps your os is not supported"
def __init__(self, urls, controllers, config=None, auth_class=WhoAuthentication): super(StorageServerApp, self).__init__(urls, controllers, config, auth_class) self.config = config # Collecting the host-specific config and building connectors. self.storages = {'default': get_storage(config)} hostnames = set() host_token = 'host:' for cfgkey in config: if cfgkey.startswith(host_token): # Get the hostname from the config key. This assumes # that host-specific keys have two trailing components # that specify the setting to override. # E.g: "host:localhost.storage.sqluri" => "localhost" hostname = cfgkey[len(host_token):].rsplit(".", 2)[0] hostnames.add(hostname) for hostname in hostnames: host_cfg = self._host_specific(hostname, config) self.storages[hostname] = get_storage(host_cfg) # If we need to check node status, then we need to # obtain a memcache client object. self.cache = None self.check_node_status = \ self.config.get('storage.check_node_status', False) if self.check_node_status: if Client is None: raise ValueError('The "check_node_status" option ' 'needs a memcached server') servers = self.config.get('storage.cache_servers', '127.0.0.1:11211') self.cache = Client(servers.split(',')) # Config knobs for managing staged rollout of EOL headers. self.eol_header_value = json.dumps({ 'code': self.config.get('storage.eol_header_code', 'soft-eol'), 'message': self.config.get('storage.eol_header_message', ''), 'url': self.config.get('storage.eol_header_url', ''), }) self.eol_general_release = \ self.config.get('storage.eol_general_release', False) self.eol_rollout_percent = \ int(self.config.get('storage.eol_rollout_percent', 0)) self.eol_rollout_percent_hwm = \ int(self.config.get('storage.eol_rollout_percent_hwm', 0)) if not 0 <= self.eol_rollout_percent <= 100: raise ValueError('storage.eol_rollout_percent should be ' \ 'between 0 and 100') if not 0 <= self.eol_rollout_percent_hwm <= 100: raise ValueError('storage.eol_rollout_percent_hwm should be ' \ 'between 0 and 100') if self.eol_rollout_percent_hwm > 0: if not self.eol_rollout_percent_hwm >= self.eol_rollout_percent: raise ValueError('storage.eol_rollout_percent_hwm should be ' \ 'greater than storage.eol_rollout_percent')
def cache(localconfig=None): "cache object" if localconfig: server = localconfig.get('baruwa.memcached.host', '127.0.0.1') else: server = config.get('baruwa.memcached.host', '127.0.0.1') beh = {"tcp_nodelay": True, "ketama": True} conn = Client([server], binary=True, behaviors=beh) return conn
def __init__(self, *, namespace="dramatiq-results", encoder=None, pool=None, pool_size=8, **parameters): super().__init__(namespace=namespace, encoder=encoder) self.pool = pool or ClientPool(Client(**parameters), pool_size)
def __init__(self, **options): self.servers = [ server.strip() for server in options['servers'].split(',') ] self.prefix = options['prefix'] self._client = Client(self.servers) self.pool = ThreadMappedPool(self._client) # using a locker to avoid race conditions # when several clients for the same user # get/set the cached data self._locker = threading.RLock()
def getmc(): """ @return: Memcached ClientPool """ try: from pylibmc import Client, ClientPool from config import cacheServers, MC_POOL_SIZE adv_setting = {"cas": True, "tcp_nodelay": True, "ketama": True} mc = Client(cacheServers, binary=True, behaviors=adv_setting) return ClientPool(mc, MC_POOL_SIZE) except ImportError: return ClientPoolEmptyOnWIndows()
def _setup_cache(app): """ If a test is being run or we don't want cache, NullCache will be initialized just as a dummy. If running locally without the 'DISABLE_CACHE' env variable and without a memcached instance running, MemcachedCache and it's underlying pylibmc will give no warning on connection, but will throw exceptions when trying to work with the cache. A few connection retires will be made in that scenario, and eventually the cache will be replaced with a NullCache. Binary communications must be used for SASL. """ # initialize the retry count if it's our first time here if not hasattr(app, 'cache_retry'): app.cache_retry = 0 # Setup cache if app.config['TESTING'] or os.environ.get('DISABLE_CACHE', None) is not None: app.cache = NullCache() app.logger.debug('Cache initialized as NullCache') else: MEMCACHED_SERVERS = os.environ.get('MEMCACHEDCLOUD_SERVERS', '127.0.0.1:11211') try: memcached_client = Client( servers=MEMCACHED_SERVERS.split(','), username=os.environ.get('MEMCACHEDCLOUD_USERNAME'), password=os.environ.get('MEMCACHEDCLOUD_PASSWORD'), binary=True) app.cache = MemcachedCache(memcached_client) app.logger.debug( 'Cache initialized as MemcachedCache with servers: %s', MEMCACHED_SERVERS) except Exception as e: # very unlikely to have an exception here. pylibmc mostly throws when trying to communicate, not connect app.logger.error('Error initializing MemcachedCache: %s', e) app.logger.error('Initializing cache as NullCache. Fix ASAP!') app.cache = NullCache()
def __init__(self, *, pool=None, pool_size=8, **parameters): behaviors = parameters.setdefault("behaviors", {}) behaviors["cas"] = True self.pool = pool or ClientPool(Client(**parameters), pool_size)
app = Flask(__name__) servers = os.environ.get('MEMCACHIER_SERVERS', '').split(',') user = os.environ.get('MEMCACHIER_USERNAME', '') passw = os.environ.get('MEMCACHIER_PASSWORD', '') # Client config obtained from https://devcenter.heroku.com/articles/memcachier#python mc = Client( servers, binary=True, username=user, password=passw, behaviors={ # Faster IO "tcp_nodelay": True, # Keep connection alive 'tcp_keepalive': True, # Timeout for set/get requests 'connect_timeout': 2000, # ms 'send_timeout': 750 * 1000, # us 'receive_timeout': 750 * 1000, # us '_poll_timeout': 2000, # ms # Better failover 'ketama': True, 'remove_failed': 1, 'retry_timeout': 2, 'dead_timeout': 30, }) from app import routes
def create_app(): """Instanciate app.""" app = Flask(__name__) if os.path.exists(app.config.root_path + '/config.cfg') is False: print("copy config.default.cfg to config.cfg and add your settings") app.config.from_pyfile(app.config.root_path + '/config.default.cfg') else: app.config.from_pyfile(app.config.root_path + '/config.cfg') babel = Babel(app) Compress(app) @babel.localeselector def get_locale(): """Get correct language from url.""" locale = request.path[1:].split('/', 1)[0] if locale in ['sv', 'en']: return locale else: locale = 'sv' for lang in list(request.accept_languages.values()): if lang[:2] in ['sv', 'en']: locale = lang[:2] break g.locale = locale return locale client = Client(app.config['MEMCACHED']) @app.before_request def func(): g.babel = Babel g.language = get_locale() g.config = app.config g.mc_pool = ClientPool(client, app.config['POOL_SIZE']) @app.context_processor def inject_custom(): d = { 'lurl_for': lambda ep, **kwargs: url_for(ep + '_' + g.language, **kwargs) } return d @app.template_filter('deescape') def deescape_filter(s): html_parser = html.parser.HTMLParser() return html_parser.unescape(s) @app.template_filter('cclink') def cclink_filter(s): return re.sub( r'(CC-BY\S*)', '<a href="https://creativecommons.org/licenses/" target="_blank">\\1</a>', s) from . import helpers app.jinja_env.globals.update(get_life_range=helpers.get_life_range) app.jinja_env.globals.update(make_namelist=helpers.make_namelist) app.jinja_env.globals.update(make_datelist=helpers.make_datelist) app.jinja_env.globals.update( make_simplenamelist=helpers.make_simplenamelist) app.jinja_env.globals.update(make_placelist=helpers.make_placelist) app.jinja_env.globals.update(make_placenames=helpers.make_placenames) app.jinja_env.globals.update( make_alphabetical_bucket=helpers.make_alphabetical_bucket) app.jinja_env.globals.update(get_date=helpers.get_date) app.jinja_env.globals.update(join_name=helpers.join_name) app.jinja_env.globals.update(swedish_translator=helpers.swedish_translator) app.jinja_env.globals.update(sorted=sorted) app.jinja_env.globals.update(len=len) app.jinja_env.globals.update(get_lang_text=helpers.get_lang_text) app.jinja_env.globals.update(get_shorttext=helpers.get_shorttext) app.jinja_env.globals.update(get_org_name=helpers.get_org_name) app.jinja_env.globals.update(rewrite_von=helpers.rewrite_von) app.jinja_env.globals.update(lowersorted=helpers.lowersorted) app.jinja_env.globals.update(get_current_date=helpers.get_current_date) app.jinja_env.globals.update(karp_fe_url=helpers.karp_fe_url) from . import views app.register_blueprint(views.bp) app.register_error_handler(Exception, views.page_not_found) app.wsgi_app = flask_reverse_proxy.ReverseProxied(app.wsgi_app) return app
def __init__(self, config): super(MemcacheCache, self).__init__() self.mc = Client(config['MEMCACHE'])
class StorageServerApp(SyncServerApp): """Storage application""" def __init__(self, urls, controllers, config=None, auth_class=WhoAuthentication): super(StorageServerApp, self).__init__(urls, controllers, config, auth_class) self.config = config # Collecting the host-specific config and building connectors. self.storages = {'default': get_storage(config)} hostnames = set() host_token = 'host:' for cfgkey in config: if cfgkey.startswith(host_token): # Get the hostname from the config key. This assumes # that host-specific keys have two trailing components # that specify the setting to override. # E.g: "host:localhost.storage.sqluri" => "localhost" hostname = cfgkey[len(host_token):].rsplit(".", 2)[0] hostnames.add(hostname) for hostname in hostnames: host_cfg = self._host_specific(hostname, config) self.storages[hostname] = get_storage(host_cfg) # If we need to check node status, then we need to # obtain a memcache client object. self.cache = None self.check_node_status = \ self.config.get('storage.check_node_status', False) if self.check_node_status: if Client is None: raise ValueError('The "check_node_status" option ' 'needs a memcached server') servers = self.config.get('storage.cache_servers', '127.0.0.1:11211') self.cache = Client(servers.split(',')) # Config knobs for managing staged rollout of EOL headers. self.eol_header_value = json.dumps({ 'code': self.config.get('storage.eol_header_code', 'soft-eol'), 'message': self.config.get('storage.eol_header_message', ''), 'url': self.config.get('storage.eol_header_url', ''), }) self.eol_general_release = \ self.config.get('storage.eol_general_release', False) self.eol_rollout_percent = \ int(self.config.get('storage.eol_rollout_percent', 0)) self.eol_rollout_percent_hwm = \ int(self.config.get('storage.eol_rollout_percent_hwm', 0)) if not 0 <= self.eol_rollout_percent <= 100: raise ValueError('storage.eol_rollout_percent should be ' \ 'between 0 and 100') if not 0 <= self.eol_rollout_percent_hwm <= 100: raise ValueError('storage.eol_rollout_percent_hwm should be ' \ 'between 0 and 100') if self.eol_rollout_percent_hwm > 0: if not self.eol_rollout_percent_hwm >= self.eol_rollout_percent: raise ValueError('storage.eol_rollout_percent_hwm should be ' \ 'greater than storage.eol_rollout_percent') def get_storage(self, request): host = request.host if host not in self.storages: host = 'default' return self.storages[host] def _before_call(self, request): headers = {} # If configured to do so, this function can check memcache for # the status of the target node and possibly avoid calling out # to the storage backend. It looks for a memcache key named # "status:<hostname>" with one of the following values: # # "down": the node is explicitly marked as down # "draining": the node is being decommissioned # "unhealthy": the node has not responded to health checks # "backoff" or "backoff:NN": the node is under heavy load and # clients should back off for a while. if self.check_node_status: # Helper function to create a HTTPServiceUnavailable response. # This takes care of some fiddly details in the API. def resp_service_unavailable(msg): body = "server issue: " + msg headers["Retry-After"] = str(self.retry_after) headers["X-Weave-Backoff"] = str(self.retry_after) raise HTTPServiceUnavailable(headers=headers, body_template=body) # Get the node name from host header, # and check that it's one we know about. node = request.host if not node: msg = "host header not received from client" raise resp_service_unavailable(msg) if node not in self.storages: msg = "database lookup failed" raise resp_service_unavailable(msg) status = self.cache.get('status:%s' % request.host) if status is not None: # If it's marked as draining then send a 503 response. # XXX TODO: consider sending a 401 to trigger migration? if status == "draining": msg = "node reassignment" raise resp_service_unavailable(msg) # If it's marked as being down then send a 503 response. if status == "down": msg = "database marked as down" raise resp_service_unavailable(msg) # If it's marked as being unhealthy then send a 503 response. if status == "unhealthy": msg = "database is not healthy" raise resp_service_unavailable(msg) # If it's marked for backoff, proceed with the request # but set appropriate headers on the response. if status == "backoff" or status.startswith("backoff:"): try: backoff = status.split(":", 1)[1] except IndexError: backoff = str(self.retry_after) headers["X-Weave-Backoff"] = backoff return headers def _debug_server(self, request): res = [] storage = self.get_storage(request) res.append('- backend: %s' % storage.get_name()) if storage.get_name() in ('memcached',): cache_servers = ['%s:%d' % (server.ip, server.port) for server in storage.cache.servers] res.append('- memcached servers: %s</li>' % ', '.join(cache_servers)) if storage.get_name() in ('sql', 'memcached'): res.append('- sqluri: %s' % storage.sqluri) return res def _dispatch_request_with_match(self, request, match): # Manage staged rollout of end-of-life headers. # We need to do it in this method rather than _before_call(), # because the logic needs access to the numeric userid. try: userid = int(request.user['userid']) except (AttributeError, KeyError): # In e.g. self-hosted environments, we may not have a userid # on all requests. Just process them normally. supercls = super(StorageServerApp, self) return supercls._dispatch_request_with_match(request, match) # We're going to test the EOL headers while the corresponding # client code is still riding the trains. Until general release, it's # only safe to show them to sufficiently-recent desktop devices. if self.eol_general_release: is_eligible_user_agent = True else: regex = USER_AGENT_WITH_EOL_SUPPORT user_agent = request.headers.get("User-Agent", "") is_eligible_user_agent = (regex.match(user_agent) is not None) # Show the EOL headers to the configured percentage of users, # based on the low-order bits of their userid. This is a simple # way to ensure that users will consistently see headers on or off. should_see_eol_headers = (is_eligible_user_agent and userid % 100 < self.eol_rollout_percent) # Before general release, we only want to show the headers to users # with a single device connected. Sadly, the only way to check this # is by doing a massive layering violation and peeking at the DB. if should_see_eol_headers and not self.eol_general_release: storage = self.get_storage(request) client_records = storage.get_items(userid, "clients", limit=2) if len(client_records) > 1: should_see_eol_headers = False # If we ever make the eol_rollout_percent smaller, it will mean that # some users stop seeing EOL headers where previously they were seeing # them. This could interfere with in-progress migrations, so we need # to check whether those users have already started the process, and # keep sending headers if so. if not should_see_eol_headers: if self.eol_rollout_percent_hwm > self.eol_rollout_percent: if userid % 100 < self.eol_rollout_percent_hwm: # Check in the DB whether this user has started migrating. # It's a massive layering violation, but gets the job done! storage = self.get_storage(request) if storage.get_item(userid, "meta", "fxa_credentials"): should_see_eol_headers = True # Log some metrics to help track progress and performance of migration: # * count of requests for which we considered sending the header # * count of requests for which we actually sent the header # * count of reads and writes to the migration sentinel # * count of client record cleanups by users who received a header if is_eligible_user_agent: self.logger.incr(CTR_EOL_HEADER_CONSIDERED) if should_see_eol_headers: self.logger.incr(CTR_EOL_HEADER_SENT) if request.method == "GET": if match.get("collection") == "meta": if match.get("item") == "fxa_credentials": self.logger.incr(CTR_EOL_SENTINEL_READ) elif request.method in ("PUT", "POST"): if match.get("collection") == "meta": if match.get("item") == "fxa_credentials": self.logger.incr(CTR_EOL_SENTINEL_WRITE) elif request.method == "DELETE": if should_see_eol_headers: self.logger.incr(CTR_EOL_CLIENT_DELETE) # Continue with request dispatch, ensuring that we set the appropriate # headers on both success and failure responses. supercls = super(StorageServerApp, self) try: response = supercls._dispatch_request_with_match(request, match) except HTTPException, response: if should_see_eol_headers: response.headers['X-Weave-Alert'] = self.eol_header_value raise else:
def __init__(self, servers, services, ttl, binary): self.services = services self.ttl = ttl self._cache = Client(servers, binary=binary) self._cache.behaviors = {"no_block": True}
from urllib2 import Request, urlopen import HTMLParser import helpers app = Flask(__name__) if os.path.exists(app.config.root_path + '/config.cfg') is False: print "copy config.default.cfg to config.cfg and add your settings" app.config.from_pyfile(app.config.root_path + '/config.default.cfg') else: app.config.from_pyfile(app.config.root_path + '/config.cfg') babel = Babel(app) Compress(app) client = Client(app.config['MEMCACHED']) mc_pool = ClientPool(client, app.config['POOL_SIZE']) def cache_name(pagename, lang=''): if not lang: lang = 'sv' if 'sv' in request.url_rule.rule else 'en' return '%s_%s' % (pagename, lang) def check_cache(page, lang=''): # If the cache should not be used, return None if app.config['TEST']: return None try: with mc_pool.reserve() as client:
def __init__(self, *, pool_size=8, **parameters): behaviors = parameters["behaviors"] = parameters.get("behaviors", {}) behaviors["cas"] = True self.client = client = Client(**parameters) self.pool = ClientPool(client, pool_size)
class StorageServerApp(SyncServerApp): """Storage application""" def __init__(self, urls, controllers, config=None, auth_class=Authentication): super(StorageServerApp, self).__init__(urls, controllers, config, auth_class) self.config = config # Collecting the host-specific config and building connectors. self.storages = {'default': get_storage(config)} hostnames = set() host_token = 'host:' for cfgkey in config: if cfgkey.startswith(host_token): # Get the hostname from the config key. This assumes # that host-specific keys have two trailing components # that specify the setting to override. # E.g: "host:localhost.storage.sqluri" => "localhost" hostname = cfgkey[len(host_token):].rsplit(".", 2)[0] hostnames.add(hostname) for hostname in hostnames: host_cfg = self._host_specific(hostname, config) self.storages[hostname] = get_storage(host_cfg) # If we need to check node status, then we need to # obtain a memcache client object. self.cache = None self.check_node_status = \ self.config.get('storage.check_node_status', False) if self.check_node_status: if Client is None: raise ValueError('The "check_node_status" option ' 'needs a memcached server') servers = self.config.get('storage.cache_servers', '127.0.0.1:11211') self.cache = Client(servers.split(',')) def get_storage(self, request): host = request.host if host not in self.storages: host = 'default' return self.storages[host] def _before_call(self, request): headers = {} # If configured to do so, this function can check memcache for # the status of the target node and possibly avoid calling out # to the storage backend. It looks for a memcache key named # "status:<hostname>" with one of the following values: # # "down": the node is explicitly marked as down # "draining": the node is being decommissioned # "unhealthy": the node has not responded to health checks # "backoff" or "backoff:NN": the node is under heavy load and # clients should back off for a while. if self.check_node_status: # Helper function to create a HTTPServiceUnavailable response. # This takes care of some fiddly details in the API. def resp_service_unavailable(msg): body = "server issue: " + msg headers["Retry-After"] = str(self.retry_after) headers["X-Weave-Backoff"] = str(self.retry_after) raise HTTPServiceUnavailable(headers=headers, body_template=body) # Get the node name from host header, # and check that it's one we know about. node = request.host if not node: msg = "host header not received from client" raise resp_service_unavailable(msg) if node not in self.storages: msg = "database lookup failed" raise resp_service_unavailable(msg) status = self.cache.get('status:%s' % request.host) if status is not None: # If it's marked as draining then send a 503 response. # XXX TODO: consider sending a 401 to trigger migration? if status == "draining": msg = "node reassignment" raise resp_service_unavailable(msg) # If it's marked as being down then send a 503 response. if status == "down": msg = "database marked as down" raise resp_service_unavailable(msg) # If it's marked as being unhealthy then send a 503 response. if status == "unhealthy": msg = "database is not healthy" raise resp_service_unavailable(msg) # If it's marked for backoff, proceed with the request # but set appropriate headers on the response. if status == "backoff" or status.startswith("backoff:"): try: backoff = status.split(":", 1)[1] except IndexError: backoff = str(self.retry_after) headers["X-Weave-Backoff"] = backoff return headers def _debug_server(self, request): res = [] storage = self.get_storage(request) res.append('- backend: %s' % storage.get_name()) if storage.get_name() in ('memcached',): cache_servers = ['%s:%d' % (server.ip, server.port) for server in storage.cache.servers] res.append('- memcached servers: %s</li>' % ', '.join(cache_servers)) if storage.get_name() in ('sql', 'memcached'): res.append('- sqluri: %s' % storage.sqluri) return res
with open(os.path.join(APP_STATIC, 'post_data_stable.xml')) as f: POST_DATA_STABLE = f.read().replace('\n', '') with open(os.path.join(APP_STATIC, 'post_data_beta.xml')) as f: POST_DATA_BETA = f.read().replace('\n', '') with open(os.path.join(APP_STATIC, 'post_data_dev.xml')) as f: POST_DATA_DEV = f.read().replace('\n', '') post_data = OrderedDict([('stable', POST_DATA_STABLE), ('beta', POST_DATA_BETA), ('dev', POST_DATA_DEV)]) if MEMCACHE_SERVERS: client = Client([MEMCACHE_SERVERS], behaviors={"tcp_nodelay": True}, binary=True, username=MEMCACHE_USERNAME, password=MEMCACHE_PASSWORD) else: client = Client(['127.0.0.1:11211']) cache = MemcachedCache(client, default_timeout=60) def cached(timeout=60): def decorator(f): @wraps(f) def decorated_function(*args, **kwargs): key = args[0] rv = cache.get(key) if rv is not None: print 'hit\t\t', key return rv print 'miss\t\t', key
class Memcached(object): implements(IMemcachedProvider) def __init__(self, servers): """Initialize memcached. @param servers: an array of servers. Servers can be passed in two forms: 1. Strings of the form host:port (implies a default weight of 1). 2. Tuples of the form (host:port, weight) (weight as integer) """ self._client = Client(servers) self.timeout = 0 def reset(self): self._client.flush_all() def size(self): bytes = 0 stats = self._client.get_stats() for name, stat in stats: bytes+=int(stat['bytes']) return bytes def keys(self): raise MemcachedException, \ "It's not possible to fetch keys from memcached" def values(self): raise MemcachedException, \ "It's not possible to fetch values from memcached" def get(self, key, default=None): value = self._client.get(key) if value is not None: return value return default def __getitem__(self, key): return self._client.get(key) def __setitem__(self, key, object): if PYLIBMC: self._client.set(key, object, time=self.timeout, min_compress_len=1024000, compress_level=zlib.Z_BEST_SPEED) else: self._client.set(key, object, time=self.timeout) def __delitem__(self, key): self._client.delete(key)