def __init__(self,
servers,
debug = False,
noreply = False,
no_block = False,
num_clients = 10,
legacy = False):
self.servers = servers
self.legacy = legacy
self.clients = pylibmc.ClientPool(n_slots = num_clients)
for x in xrange(num_clients):
client = pylibmc.Client(servers, binary=not self.legacy)
behaviors = {
'no_block': no_block, # use async I/O
'cache_lookups': True, # cache DNS lookups
'tcp_nodelay': True, # no nagle
'_noreply': int(noreply),
'verify_key': int(debug), # spend the CPU to verify keys
}
# so that we can drop this in to share keys with
# python-memcached, we have a 'legacy' mode that MD5s the
# keys and uses the old CRC32-modula mapping
if self.legacy:
behaviors['hash'] = 'crc'
else:
behaviors['ketama'] = True # consistent hashing
client.behaviors.update(behaviors)
self.clients.put(client)
self.min_compress_len = 512*1024
def connect(self):
mc = pylibmc.Client(self.SERVERS,
binary=True,
behaviors={
'tcp_nodelay': True,
'no_block': True,
'ketama': True
})
return pylibmc.ClientPool(mc, self.POOL_SIZE)
def __init__(self,
servers,
debug=False,
noreply=False,
no_block=False,
num_clients=10):
self.servers = servers
self.clients = pylibmc.ClientPool(n_slots=num_clients)
for x in xrange(num_clients):
client = pylibmc.Client(servers, binary=True)
behaviors = {
'no_block': no_block, # use async I/O
'tcp_nodelay': True, # no nagle
'_noreply': int(noreply),
'ketama': True, # consistent hashing
}
client.behaviors.update(behaviors)
self.clients.put(client)
self.min_compress_len = 512 * 1024
def create_recommendation_microservice(self,
recommender_folder,
memcache_servers=None,
memcache_pool_size=2):
"""
create recommedation Flask microservice app
Parameters
----------
recommender_folder : str
location of recommender model files
memcache_servers : comma separated string, optional
memcache server locations, e.g., 127.0.0.1:11211
memcache_pool_size : int, optional
size of memcache pool
"""
app = Flask(__name__)
if not memcache_servers is None:
mc = pylibmc.Client(memcache_servers)
_mc_pool = pylibmc.ClientPool(mc, memcache_pool_size)
app.config["seldon_memcache"] = _mc_pool
if self.aws_key:
rw = seldon.RecommenderWrapper(aws_key=self.aws_key,
aws_secret=self.aws_secret)
else:
rw = seldon.RecommenderWrapper()
recommender = rw.load_recommender(recommender_folder)
app.config["seldon_recommender"] = recommender
app.register_blueprint(recommend_blueprint)
# other setup tasks
return app
import random
import subprocess
import sys
import time
import celery
import pylibmc
import remoulade
from remoulade.brokers.rabbitmq import RabbitmqBroker
from remoulade.brokers.redis import RedisBroker
logger = logging.getLogger("example")
counter_key = "latench-bench-counter"
memcache_client = pylibmc.Client(["localhost"], binary=True)
memcache_pool = pylibmc.ClientPool(memcache_client, 8)
random.seed(1337)
if os.getenv("REDIS") == "1":
broker = RedisBroker()
remoulade.set_broker(broker)
celery_app = celery.Celery(broker="redis:///")
else:
broker = RabbitmqBroker(host="127.0.0.1")
remoulade.set_broker(broker)
celery_app = celery.Celery(broker="amqp:///")
def fib_bench(n):
p, q = 0, 1
def test_exhaust(self):
p = pylibmc.ClientPool(self.mc, 2)
with p.reserve() as smc1:
with p.reserve() as smc2:
self.assertRaises(queue.Empty, p.reserve().__enter__)
from config import config
import locale
import pylibmc
import threading
import sys
import time
PySQLPool.getNewPool().maxActiveConnections = 100
PySQLPool.getNewPool().maxActivePerConnection = 1
mc = pylibmc.Client(["127.0.0.1"],
binary=True,
behaviors={
"tcp_nodelay": True,
"ketama": True
})
pool = pylibmc.ClientPool()
db = PySQLPool.getNewConnection(user=config['username'],
passwd=config['password'],
db=config['db'])
locale.setlocale(locale.LC_ALL, 'en_US')
maxThreads = 60
pool.fill(mc, maxThreads + 10)
repoList = []
repoVal = {}
def repoThread():
global repoList
global repoVal
while len(repoList) > 0:
row = repoList.pop()
def mc_init():
global _mc_pool
mc_servers = app.config['MEMCACHE']['servers']
mc_pool_size = app.config['MEMCACHE']['pool_size']
mc = pylibmc.Client(mc_servers)
_mc_pool = pylibmc.ClientPool(mc, mc_pool_size)
def test_simple(self):
p = pylibmc.ClientPool(self.mc, 2)
with p.reserve() as smc:
ok_(smc)
ok_(smc.set("a", 1))
eq_(smc["a"], 1)
