def main():
mc1 = libmc.Client(['localhost:%s' % PORT])
mc1.config(libmc.MC_POLL_TIMEOUT, BLOCKING_SECONDS * 1000 * 2)
assert mc1.version(), "start slow_memcached_server first"
mc1.config(libmc.MC_POLL_TIMEOUT, BLOCKING_SECONDS * 1000 / 2)
assert not mc1.set('foo', 1)
mc2 = libmc.Client(['localhost:%s' % PORT])
mc2.config(libmc.MC_POLL_TIMEOUT, BLOCKING_SECONDS * 1000 * 2)
assert not mc1.set('foo', 1)
def store_synonyms(synonyms_input_dir):
client = libmc.Client(["127.0.0.1:11219"],
comp_threshold=0,
noreply=False,
prefix=None,
hash_fn=MC_HASH_MD5,
failover=False)
client.config(MC_POLL_TIMEOUT, 100) # 100ms
client.config(MC_CONNECT_TIMEOUT, 300) # 300ms
client.config(MC_RETRY_TIMEOUT, 5) # 5s
path = synonyms_input_dir
for filename in glob.glob(path):
print "input data file:", filename
with open(filename, 'r') as f:
for line in f:
entry = json.loads(line.strip())
key = entry['word']
fields = entry['synonyms']
#print type(fields)
val = str(key.encode('utf-8'))
for i in range(0, len(fields)):
val = val + "_" + fields[i].encode('utf-8').strip('""')
#save to memcached
client.set(key, val)
print "key=", key
print "val=", val
def main(argv):
if len(argv) != 2:
print >> sys.stderr, "Usage: python %s hostname:11211" % argv[0]
return 1
s = argv[1]
if s.isdigit():
server_addr = "%s:%s" % ('127.0.0.1', s)
else:
server_addr = s
mc = libmc.Client([server_addr])
noise_data = []
for key_prefix in ('foo', 'bar', 'baz'):
for value_size in (1, 10, 100, 1000, 10000):
key = '%s_with_value_size_%s' % (key_prefix, value_size)
val = '1' * value_size
mc.set(key, val)
noise_data.append((key, val))
for (key, val) in itertools.cycle(noise_data):
val2 = mc.get(key)
assert val2 == val
mc.set(key, val)
time.sleep(0.5)
def main():
# NOTE start 2 slow memcached server first!
mc = libmc.Client(['localhost:8965', 'localhost:8966'])
mc.config(libmc.MC_POLL_TIMEOUT, int(UNIT_PENDING_SECONDS * 0.5))
bv = '1' * 512
assert not mc.set('bv', bv)
assert mc.get('bv') is None
assert not mc.set('foo', 1)
assert mc.get('stubs') is None
mc.config(libmc.MC_POLL_TIMEOUT, int(UNIT_PENDING_SECONDS * 1.2))
assert mc.set('foo', 1)
assert mc.get('stubs') == 'yes'
assert mc.set('bv', bv)
assert mc.get('bv') is None # this should be timed out
assert mc.get_multi(['stubs']) == {'stubs': 'yes'}
assert mc.get_multi(['stubs', 'foo']) == {'stubs': 'yes', 'foo': 1}
assert not mc.get_multi(['bv']) # this should be timed out
# NOTE: A C++ backtrace will be printed to stderr if reproduced
assert mc.get_multi(['bv', 'stubs']) == {
'stubs': 'yes'
} # this should be timed out
assert mc.set('foo', 1)
assert mc.get_multi(['stubs', 'foo']) == {'stubs': 'yes', 'foo': 1}
def Client(servers=None, *args, **kwargs):
key = hashdict([servers, kwargs])
mc = _clients.get(key)
if not mc:
mc = libmc.Client(servers, *args, **kwargs)
_clients[key] = mc
return mc
def __init__(self, config):
self.dirs = config['kenshin']['directories']
self.carbonlink = CarbonLinkPool(config['kenshin']['carbonlink_hosts'])
global EXPIRE_TIME, mc
EXPIRE_TIME = int(config['kenshin']['memcached']['expire_time'])
mc = libmc.Client(config['kenshin']['memcached']['hosts'])
def store_tf(feature_dir):
client = libmc.Client(["127.0.0.1:11220"],
comp_threshold=0,
noreply=False,
prefix=None,
hash_fn=MC_HASH_MD5,
failover=False)
client.config(MC_POLL_TIMEOUT, 100) # 100 ms
client.config(MC_CONNECT_TIMEOUT, 300) # 300 ms
client.config(MC_RETRY_TIMEOUT, 5) # 5 s
path = feature_dir + "/part*"
for filename in glob.glob(path):
print "input data file:", filename
with open(filename, 'r') as f:
for line in f:
entry = json.loads(line.strip())
#print "input line:",line
key = entry['adid_terms']
#print key
val = entry['count']
#print val
client.set(key, val)
print "key=", key
print "val=", val
def import_preferred_memcache_lib(self, servers):
"""Returns an initialized memcache client. Used by the constructor."""
try:
import pylibmc
except ImportError:
pass
else:
return pylibmc.Client(servers)
try:
from google.appengine.api import memcache
except ImportError:
pass
else:
return memcache.Client()
try:
import memcache
except ImportError:
pass
else:
return memcache.Client(servers)
try:
import libmc
except ImportError:
pass
else:
return libmc.Client(servers)
def get_memcache_connection(self, servers):
try:
import pylibmc
except ImportError:
print("=== 引入pylibmc出错 ===")
else:
return pylibmc.Client(servers)
try:
from google.appengine.api import memcache
except ImportError:
print("=== 引入google出错 ===")
else:
return memcache.Client()
try:
import memcache
except ImportError:
print("=== 引入memcache出错 ===")
else:
return memcache.Client(servers)
try:
import libmc
except ImportError:
print("=== 引入libmc出错 ===")
else:
return libmc.Client(servers)
def test_stats(self):
s = self.mc.stats()
self.assertEqual(TEST_SERVER in s, True)
st = s[TEST_SERVER]
st_keys = sorted([
"pid",
"uptime",
"time",
"version",
"pointer_size",
"rusage_user",
"rusage_system",
"curr_items",
"total_items",
"bytes",
"curr_connections",
"total_connections",
"connection_structures",
"cmd_get",
"cmd_set",
"get_hits",
"get_misses",
"evictions",
"bytes_read",
"bytes_written",
"limit_maxbytes",
"threads",
])
self.assertEqual(sorted(st.keys()), st_keys)
mc = libmc.Client(["localhost:11999", TEST_SERVER])
s = mc.stats()
self.assertEqual(len(s), 2)
def test_last_error(self):
self.assertEqual(self.mc.set('testkey', 'hh'), True)
self.assertEqual(self.mc.get('testkey'), 'hh')
self.assertEqual(self.mc.get_last_error(), 0)
self.mc = libmc.Client(["localhost:11999"], comp_threshold=1024)
self.assertEqual(self.mc.set('testkey', 'hh'), False)
self.assertEqual(self.mc.get('testkey'), None)
self.assertNotEqual(self.mc.get_last_error(), 1)
def __init__(self, host_alias):
self.host_alias = host_alias
self.host = socket.gethostbyname_ex(self.host_alias)[0]
self.addr = self.host + ":7905"
self.node = Node(self.addr)
self.web = self.node.web_client()
self.server_addr = '%s:%s' % (self.host, PROXY_SERVER_PORT)
self.web_addr = '%s:%s' % (self.host, PROXY_WEB_PORT)
self.server = libmc.Client([self.server_addr])
def __init__(self, config=None):
if config is not None:
self.dirs = config['kenshin']['directories']
self.carbonlink = CarbonLinkPool(
config['kenshin']['carbonlink_hosts'])
global EXPIRE_TIME, mc
EXPIRE_TIME = int(config['kenshin']['memcached']['expire_time'])
mc = libmc.Client(config['kenshin']['memcached']['hosts'])
else:
from django.conf import settings
self.dirs = getattr(settings, 'KENSHIN_DIRECTORIES')
self.carbonlink = CarbonLinkPool(
getattr(settings, 'KENSHIN_CARBONLINK_HOSTS'))
self.index_files = getattr(settings, 'KENSHIN_INDEX_FILES')
global EXPIRE_TIME, mc
EXPIRE_TIME = int(
getattr(settings, 'KENSHIN_MEMCACHED_EXPIRE_TIME'))
mc = libmc.Client(getattr(settings, 'KENSHIN_MEMCACHED_HOSTS'))
def test_set_multi(self):
values = dict(
('key%s' % k, ('value%s' % k) * 100000 * k) for k in range(10))
values.update({' ': ''})
self.assertEqual(self.mc.set_multi(values), 1)
del values[' ']
for k in values:
self.assertEqual(self.mc.get(k), values[k])
mc = libmc.Client(["localhost:11999"], comp_threshold=1024)
self.assertEqual(mc.set_multi(values), 0)
def connect(server, **kwargs):
comp_threshold = kwargs.pop('comp_threshold', 0)
prefix = kwargs.pop('prefix', None)
c = libmc.Client([server],
do_split=0,
comp_threshold=comp_threshold,
prefix=prefix)
c.config(libmc.MC_CONNECT_TIMEOUT, 300) # 0.3s
c.config(libmc.MC_POLL_TIMEOUT, 3000) # 3s
c.config(libmc.MC_RETRY_TIMEOUT, 5) # 5s
return c
def main():
print 'libmc path: %s' % libmc.__file__
mc = libmc.Client(['localhost:%s' % slow_memcached_server.PORT])
mc.config(libmc.MC_POLL_TIMEOUT, 3000)
assert mc.version(), "start slow_memcached_server first"
EXPECTED_TIMEOUT = 300
mc.config(libmc.MC_POLL_TIMEOUT, EXPECTED_TIMEOUT) # timeout in 300 ms
t0 = time.time()
assert not mc.set('foo', 1)
t1 = time.time()
error_threshold = 10
assert abs(EXPECTED_TIMEOUT - (t1 - t0) * 1000) < error_threshold
def test_hard_server_error():
normal_port = 21211
mc = libmc.Client(["127.0.0.1:%d" % normal_port])
RETRY_TIMEOUT = 20
mc.config(libmc.MC_RETRY_TIMEOUT, RETRY_TIMEOUT)
assert mc.set('foo', 1)
memcached_server_ctl('stop', normal_port)
assert not mc.set('foo', 1) # fail
memcached_server_ctl('start', normal_port)
assert not mc.set('foo', 1) # still fail
time.sleep(RETRY_TIMEOUT + 1)
assert mc.set('foo', 1) # back to live
def connect(server, **kwargs):
comp_threshold = kwargs.pop('comp_threshold', 0)
prefix = kwargs.pop('prefix', None)
if prefix is not None:
warnings.warn('"prefix" is deprecated. '
'use douban.wrapper.Prefix instead.')
c = libmc.Client([server],
do_split=0,
comp_threshold=comp_threshold,
prefix=prefix)
c.config(libmc.MC_CONNECT_TIMEOUT, 300) # 0.3s
c.config(libmc.MC_POLL_TIMEOUT, 3000) # 3s
c.config(libmc.MC_RETRY_TIMEOUT, 5) # 5s
return c
def test_soft_server_error():
mc = libmc.Client(["127.0.0.1:%d" % slow_memcached_server.PORT])
mc.config(libmc._client.MC_POLL_TIMEOUT,
slow_memcached_server.BLOCKING_SECONDS * 1000 * 2) # ms
RETRY_TIMEOUT = 2
mc.config(libmc.MC_RETRY_TIMEOUT, RETRY_TIMEOUT)
assert mc.set('foo', 1)
assert not mc.set(slow_memcached_server.KEY_SET_SERVER_ERROR, 1)
assert mc.set('foo', 1) # back to live
time.sleep(RETRY_TIMEOUT / 2)
assert mc.set('foo', 1) # alive
time.sleep(RETRY_TIMEOUT + 1)
assert mc.set('foo', 1) # alive
import sys
import libmc
import inspect
import msgpack
import functools
from cheapflight.config import DevelopmentConfig
mc = libmc.Client(DevelopmentConfig.MEMCACHED_SERVERS)
IGNORED_MC_RETURNS = {
libmc.MC_RETURN_OK,
libmc.MC_RETURN_INVALID_KEY_ERR,
}
DEFAULT_EXPIRE_IN = 3600 * 24
class cache(object):
def __init__(self, key_tmpl, expire_in=DEFAULT_EXPIRE_IN):
self.key_tmpl = key_tmpl
self.expire_in = expire_in
def __call__(self, fn):
@functools.wraps(fn)
def _(*args, **kwargs):
vv = inspect.getcallargs(fn, *args, **kwargs)
key = self.key_tmpl.format(**vv)
cached = mc.get(key)
if cached is not None:
return msgpack.loads(cached)
rv = fn(*args, **kwargs)
if rv is not None:
mc.set(key, msgpack.dumps(rv), self.expire_in)
SCHEMA_FILE = 'schema.sql'
BEANSDB_CFG = {
'localhost:11211': range(16),
}
MYSQL_HOST = 'localhost'
MYSQL_PORT = 3306
MYSQL_USER = os.getenv('MYSQL_USER', 'root')
MYSQL_PASSWORD = os.getenv('MYSQL_PASSWORD', 'root')
MYSQL_DB = 'test_olo'
MYSQL_CHARSET = 'utf8mb4'
created_tables = False
logger = logging.getLogger(__name__)
mc = libmc.Client(['localhost:11211'])
beansdb = BeansDBProxy(BEANSDB_CFG)
mysql_conn = None
approot = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..')
dbdir = os.path.join(approot, 'databases')
schema_path = os.path.join(dbdir, SCHEMA_FILE)
def init_tables():
setup_mysql_conn()
cur = mysql_conn.cursor()
tables = get_all_tables(cur)
if not tables:
create_tables(cur)
import libmc
mc = libmc.Client(['127.0.0.1:7900'])
res = mc.set("foo", "bar")
print(res)
value = mc.get("foo")
print(value)
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
import sys
import json
import random
from sets import Set
import libmc
from libmc import (
MC_HASH_MD5, MC_POLL_TIMEOUT, MC_CONNECT_TIMEOUT, MC_RETRY_TIMEOUT
)
mc = libmc.Client(
["127.0.0.1:11219"],comp_threshold=0, noreply=False, prefix=None,hash_fn=MC_HASH_MD5, failover=False
)
mc.config(MC_POLL_TIMEOUT, 100) # 100 ms
mc.config(MC_CONNECT_TIMEOUT, 300) # 300 ms
mc.config(MC_RETRY_TIMEOUT, 5) # 5 s
#input： [nike, running, shoes] , Dict
#output: all rewrite query
def query_rewriter_helper(query_terms, synonyms_dict):
if (len(query_terms) == 0):
return []
if (len(query_terms) == 1):
if query_terms[0] not in synonyms_dict:
return [query_terms[0]]
else:
return list(synonyms_dict[query_terms[0]])
prev = query_rewriter_helper(query_terms[:-1], synonyms_dict)
import gevent
import time
import gevent.monkey
import slow_memcached_server
gevent.monkey.patch_time()
import greenify
greenify.greenify()
import libmc
for so_path in libmc.DYNAMIC_LIBRARIES:
assert greenify.patch_lib(so_path)
mc = libmc.Client(["127.0.0.1:%d" % slow_memcached_server.PORT])
mc.config(libmc._client.MC_POLL_TIMEOUT,
slow_memcached_server.BLOCKING_SECONDS * 1000 * 2) # ms
stack = []
def mc_sleep():
print 'begin mc sleep'
stack.append('mc_sleep_begin')
assert mc.set('foo', 'bar')
stack.append('mc_sleep_end')
print 'end mc sleep'
def singer():
i = 0
for i in range(6):
i += 1
print '[%d] Oh, jingle bells, jingle bells, Jingle all the way.' % i
def get_mc():
return libmc.Client(MEMCACHED_HOSTS, **MEMCACHED_CONFIG)
def setUp(self):
self.mc = libmc.Client([TEST_SERVER], comp_threshold=1024)
def setUp(self):
self.mc = libmc.Client([TEST_SERVER], comp_threshold=1024)
self.mc.set_behavior(libmc.BEHAVIOR_BINARY_PROTOCOL, 1)
