def __init__(self, servers, debug = False, noreply = False, no_block = False, num_clients = 10, legacy = False): self.servers = servers self.legacy = legacy self.clients = pylibmc.ClientPool(n_slots = num_clients) for x in xrange(num_clients): client = pylibmc.Client(servers, binary=not self.legacy) behaviors = { 'no_block': no_block, # use async I/O 'cache_lookups': True, # cache DNS lookups 'tcp_nodelay': True, # no nagle '_noreply': int(noreply), 'verify_key': int(debug), # spend the CPU to verify keys } # so that we can drop this in to share keys with # python-memcached, we have a 'legacy' mode that MD5s the # keys and uses the old CRC32-modula mapping if self.legacy: behaviors['hash'] = 'crc' else: behaviors['ketama'] = True # consistent hashing client.behaviors.update(behaviors) self.clients.put(client) self.min_compress_len = 512*1024
def connect(self): mc = pylibmc.Client(self.SERVERS, binary=True, behaviors={ 'tcp_nodelay': True, 'no_block': True, 'ketama': True }) return pylibmc.ClientPool(mc, self.POOL_SIZE)
def __init__(self, servers, debug=False, noreply=False, no_block=False, num_clients=10): self.servers = servers self.clients = pylibmc.ClientPool(n_slots=num_clients) for x in xrange(num_clients): client = pylibmc.Client(servers, binary=True) behaviors = { 'no_block': no_block, # use async I/O 'tcp_nodelay': True, # no nagle '_noreply': int(noreply), 'ketama': True, # consistent hashing } client.behaviors.update(behaviors) self.clients.put(client) self.min_compress_len = 512 * 1024
def create_recommendation_microservice(self, recommender_folder, memcache_servers=None, memcache_pool_size=2): """ create recommedation Flask microservice app Parameters ---------- recommender_folder : str location of recommender model files memcache_servers : comma separated string, optional memcache server locations, e.g., 127.0.0.1:11211 memcache_pool_size : int, optional size of memcache pool """ app = Flask(__name__) if not memcache_servers is None: mc = pylibmc.Client(memcache_servers) _mc_pool = pylibmc.ClientPool(mc, memcache_pool_size) app.config["seldon_memcache"] = _mc_pool if self.aws_key: rw = seldon.RecommenderWrapper(aws_key=self.aws_key, aws_secret=self.aws_secret) else: rw = seldon.RecommenderWrapper() recommender = rw.load_recommender(recommender_folder) app.config["seldon_recommender"] = recommender app.register_blueprint(recommend_blueprint) # other setup tasks return app
import random import subprocess import sys import time import celery import pylibmc import remoulade from remoulade.brokers.rabbitmq import RabbitmqBroker from remoulade.brokers.redis import RedisBroker logger = logging.getLogger("example") counter_key = "latench-bench-counter" memcache_client = pylibmc.Client(["localhost"], binary=True) memcache_pool = pylibmc.ClientPool(memcache_client, 8) random.seed(1337) if os.getenv("REDIS") == "1": broker = RedisBroker() remoulade.set_broker(broker) celery_app = celery.Celery(broker="redis:///") else: broker = RabbitmqBroker(host="127.0.0.1") remoulade.set_broker(broker) celery_app = celery.Celery(broker="amqp:///") def fib_bench(n): p, q = 0, 1
def test_exhaust(self): p = pylibmc.ClientPool(self.mc, 2) with p.reserve() as smc1: with p.reserve() as smc2: self.assertRaises(queue.Empty, p.reserve().__enter__)
from config import config import locale import pylibmc import threading import sys import time PySQLPool.getNewPool().maxActiveConnections = 100 PySQLPool.getNewPool().maxActivePerConnection = 1 mc = pylibmc.Client(["127.0.0.1"], binary=True, behaviors={ "tcp_nodelay": True, "ketama": True }) pool = pylibmc.ClientPool() db = PySQLPool.getNewConnection(user=config['username'], passwd=config['password'], db=config['db']) locale.setlocale(locale.LC_ALL, 'en_US') maxThreads = 60 pool.fill(mc, maxThreads + 10) repoList = [] repoVal = {} def repoThread(): global repoList global repoVal while len(repoList) > 0: row = repoList.pop()
def mc_init(): global _mc_pool mc_servers = app.config['MEMCACHE']['servers'] mc_pool_size = app.config['MEMCACHE']['pool_size'] mc = pylibmc.Client(mc_servers) _mc_pool = pylibmc.ClientPool(mc, mc_pool_size)
def test_simple(self): p = pylibmc.ClientPool(self.mc, 2) with p.reserve() as smc: ok_(smc) ok_(smc.set("a", 1)) eq_(smc["a"], 1)