def main(): mc1 = libmc.Client(['localhost:%s' % PORT]) mc1.config(libmc.MC_POLL_TIMEOUT, BLOCKING_SECONDS * 1000 * 2) assert mc1.version(), "start slow_memcached_server first" mc1.config(libmc.MC_POLL_TIMEOUT, BLOCKING_SECONDS * 1000 / 2) assert not mc1.set('foo', 1) mc2 = libmc.Client(['localhost:%s' % PORT]) mc2.config(libmc.MC_POLL_TIMEOUT, BLOCKING_SECONDS * 1000 * 2) assert not mc1.set('foo', 1)
def store_synonyms(synonyms_input_dir): client = libmc.Client(["127.0.0.1:11219"], comp_threshold=0, noreply=False, prefix=None, hash_fn=MC_HASH_MD5, failover=False) client.config(MC_POLL_TIMEOUT, 100) # 100ms client.config(MC_CONNECT_TIMEOUT, 300) # 300ms client.config(MC_RETRY_TIMEOUT, 5) # 5s path = synonyms_input_dir for filename in glob.glob(path): print "input data file:", filename with open(filename, 'r') as f: for line in f: entry = json.loads(line.strip()) key = entry['word'] fields = entry['synonyms'] #print type(fields) val = str(key.encode('utf-8')) for i in range(0, len(fields)): val = val + "_" + fields[i].encode('utf-8').strip('""') #save to memcached client.set(key, val) print "key=", key print "val=", val
def main(argv): if len(argv) != 2: print >> sys.stderr, "Usage: python %s hostname:11211" % argv[0] return 1 s = argv[1] if s.isdigit(): server_addr = "%s:%s" % ('127.0.0.1', s) else: server_addr = s mc = libmc.Client([server_addr]) noise_data = [] for key_prefix in ('foo', 'bar', 'baz'): for value_size in (1, 10, 100, 1000, 10000): key = '%s_with_value_size_%s' % (key_prefix, value_size) val = '1' * value_size mc.set(key, val) noise_data.append((key, val)) for (key, val) in itertools.cycle(noise_data): val2 = mc.get(key) assert val2 == val mc.set(key, val) time.sleep(0.5)
def main(): # NOTE start 2 slow memcached server first! mc = libmc.Client(['localhost:8965', 'localhost:8966']) mc.config(libmc.MC_POLL_TIMEOUT, int(UNIT_PENDING_SECONDS * 0.5)) bv = '1' * 512 assert not mc.set('bv', bv) assert mc.get('bv') is None assert not mc.set('foo', 1) assert mc.get('stubs') is None mc.config(libmc.MC_POLL_TIMEOUT, int(UNIT_PENDING_SECONDS * 1.2)) assert mc.set('foo', 1) assert mc.get('stubs') == 'yes' assert mc.set('bv', bv) assert mc.get('bv') is None # this should be timed out assert mc.get_multi(['stubs']) == {'stubs': 'yes'} assert mc.get_multi(['stubs', 'foo']) == {'stubs': 'yes', 'foo': 1} assert not mc.get_multi(['bv']) # this should be timed out # NOTE: A C++ backtrace will be printed to stderr if reproduced assert mc.get_multi(['bv', 'stubs']) == { 'stubs': 'yes' } # this should be timed out assert mc.set('foo', 1) assert mc.get_multi(['stubs', 'foo']) == {'stubs': 'yes', 'foo': 1}
def Client(servers=None, *args, **kwargs): key = hashdict([servers, kwargs]) mc = _clients.get(key) if not mc: mc = libmc.Client(servers, *args, **kwargs) _clients[key] = mc return mc
def __init__(self, config): self.dirs = config['kenshin']['directories'] self.carbonlink = CarbonLinkPool(config['kenshin']['carbonlink_hosts']) global EXPIRE_TIME, mc EXPIRE_TIME = int(config['kenshin']['memcached']['expire_time']) mc = libmc.Client(config['kenshin']['memcached']['hosts'])
def store_tf(feature_dir): client = libmc.Client(["127.0.0.1:11220"], comp_threshold=0, noreply=False, prefix=None, hash_fn=MC_HASH_MD5, failover=False) client.config(MC_POLL_TIMEOUT, 100) # 100 ms client.config(MC_CONNECT_TIMEOUT, 300) # 300 ms client.config(MC_RETRY_TIMEOUT, 5) # 5 s path = feature_dir + "/part*" for filename in glob.glob(path): print "input data file:", filename with open(filename, 'r') as f: for line in f: entry = json.loads(line.strip()) #print "input line:",line key = entry['adid_terms'] #print key val = entry['count'] #print val client.set(key, val) print "key=", key print "val=", val
def import_preferred_memcache_lib(self, servers): """Returns an initialized memcache client. Used by the constructor.""" try: import pylibmc except ImportError: pass else: return pylibmc.Client(servers) try: from google.appengine.api import memcache except ImportError: pass else: return memcache.Client() try: import memcache except ImportError: pass else: return memcache.Client(servers) try: import libmc except ImportError: pass else: return libmc.Client(servers)
def get_memcache_connection(self, servers): try: import pylibmc except ImportError: print("=== 引入pylibmc出错 ===") else: return pylibmc.Client(servers) try: from google.appengine.api import memcache except ImportError: print("=== 引入google出错 ===") else: return memcache.Client() try: import memcache except ImportError: print("=== 引入memcache出错 ===") else: return memcache.Client(servers) try: import libmc except ImportError: print("=== 引入libmc出错 ===") else: return libmc.Client(servers)
def test_stats(self): s = self.mc.stats() self.assertEqual(TEST_SERVER in s, True) st = s[TEST_SERVER] st_keys = sorted([ "pid", "uptime", "time", "version", "pointer_size", "rusage_user", "rusage_system", "curr_items", "total_items", "bytes", "curr_connections", "total_connections", "connection_structures", "cmd_get", "cmd_set", "get_hits", "get_misses", "evictions", "bytes_read", "bytes_written", "limit_maxbytes", "threads", ]) self.assertEqual(sorted(st.keys()), st_keys) mc = libmc.Client(["localhost:11999", TEST_SERVER]) s = mc.stats() self.assertEqual(len(s), 2)
def test_last_error(self): self.assertEqual(self.mc.set('testkey', 'hh'), True) self.assertEqual(self.mc.get('testkey'), 'hh') self.assertEqual(self.mc.get_last_error(), 0) self.mc = libmc.Client(["localhost:11999"], comp_threshold=1024) self.assertEqual(self.mc.set('testkey', 'hh'), False) self.assertEqual(self.mc.get('testkey'), None) self.assertNotEqual(self.mc.get_last_error(), 1)
def __init__(self, host_alias): self.host_alias = host_alias self.host = socket.gethostbyname_ex(self.host_alias)[0] self.addr = self.host + ":7905" self.node = Node(self.addr) self.web = self.node.web_client() self.server_addr = '%s:%s' % (self.host, PROXY_SERVER_PORT) self.web_addr = '%s:%s' % (self.host, PROXY_WEB_PORT) self.server = libmc.Client([self.server_addr])
def __init__(self, config=None): if config is not None: self.dirs = config['kenshin']['directories'] self.carbonlink = CarbonLinkPool( config['kenshin']['carbonlink_hosts']) global EXPIRE_TIME, mc EXPIRE_TIME = int(config['kenshin']['memcached']['expire_time']) mc = libmc.Client(config['kenshin']['memcached']['hosts']) else: from django.conf import settings self.dirs = getattr(settings, 'KENSHIN_DIRECTORIES') self.carbonlink = CarbonLinkPool( getattr(settings, 'KENSHIN_CARBONLINK_HOSTS')) self.index_files = getattr(settings, 'KENSHIN_INDEX_FILES') global EXPIRE_TIME, mc EXPIRE_TIME = int( getattr(settings, 'KENSHIN_MEMCACHED_EXPIRE_TIME')) mc = libmc.Client(getattr(settings, 'KENSHIN_MEMCACHED_HOSTS'))
def test_set_multi(self): values = dict( ('key%s' % k, ('value%s' % k) * 100000 * k) for k in range(10)) values.update({' ': ''}) self.assertEqual(self.mc.set_multi(values), 1) del values[' '] for k in values: self.assertEqual(self.mc.get(k), values[k]) mc = libmc.Client(["localhost:11999"], comp_threshold=1024) self.assertEqual(mc.set_multi(values), 0)
def connect(server, **kwargs): comp_threshold = kwargs.pop('comp_threshold', 0) prefix = kwargs.pop('prefix', None) c = libmc.Client([server], do_split=0, comp_threshold=comp_threshold, prefix=prefix) c.config(libmc.MC_CONNECT_TIMEOUT, 300) # 0.3s c.config(libmc.MC_POLL_TIMEOUT, 3000) # 3s c.config(libmc.MC_RETRY_TIMEOUT, 5) # 5s return c
def main(): print 'libmc path: %s' % libmc.__file__ mc = libmc.Client(['localhost:%s' % slow_memcached_server.PORT]) mc.config(libmc.MC_POLL_TIMEOUT, 3000) assert mc.version(), "start slow_memcached_server first" EXPECTED_TIMEOUT = 300 mc.config(libmc.MC_POLL_TIMEOUT, EXPECTED_TIMEOUT) # timeout in 300 ms t0 = time.time() assert not mc.set('foo', 1) t1 = time.time() error_threshold = 10 assert abs(EXPECTED_TIMEOUT - (t1 - t0) * 1000) < error_threshold
def test_hard_server_error(): normal_port = 21211 mc = libmc.Client(["127.0.0.1:%d" % normal_port]) RETRY_TIMEOUT = 20 mc.config(libmc.MC_RETRY_TIMEOUT, RETRY_TIMEOUT) assert mc.set('foo', 1) memcached_server_ctl('stop', normal_port) assert not mc.set('foo', 1) # fail memcached_server_ctl('start', normal_port) assert not mc.set('foo', 1) # still fail time.sleep(RETRY_TIMEOUT + 1) assert mc.set('foo', 1) # back to live
def connect(server, **kwargs): comp_threshold = kwargs.pop('comp_threshold', 0) prefix = kwargs.pop('prefix', None) if prefix is not None: warnings.warn('"prefix" is deprecated. ' 'use douban.wrapper.Prefix instead.') c = libmc.Client([server], do_split=0, comp_threshold=comp_threshold, prefix=prefix) c.config(libmc.MC_CONNECT_TIMEOUT, 300) # 0.3s c.config(libmc.MC_POLL_TIMEOUT, 3000) # 3s c.config(libmc.MC_RETRY_TIMEOUT, 5) # 5s return c
def test_soft_server_error(): mc = libmc.Client(["127.0.0.1:%d" % slow_memcached_server.PORT]) mc.config(libmc._client.MC_POLL_TIMEOUT, slow_memcached_server.BLOCKING_SECONDS * 1000 * 2) # ms RETRY_TIMEOUT = 2 mc.config(libmc.MC_RETRY_TIMEOUT, RETRY_TIMEOUT) assert mc.set('foo', 1) assert not mc.set(slow_memcached_server.KEY_SET_SERVER_ERROR, 1) assert mc.set('foo', 1) # back to live time.sleep(RETRY_TIMEOUT / 2) assert mc.set('foo', 1) # alive time.sleep(RETRY_TIMEOUT + 1) assert mc.set('foo', 1) # alive
import sys import libmc import inspect import msgpack import functools from cheapflight.config import DevelopmentConfig mc = libmc.Client(DevelopmentConfig.MEMCACHED_SERVERS) IGNORED_MC_RETURNS = { libmc.MC_RETURN_OK, libmc.MC_RETURN_INVALID_KEY_ERR, } DEFAULT_EXPIRE_IN = 3600 * 24 class cache(object): def __init__(self, key_tmpl, expire_in=DEFAULT_EXPIRE_IN): self.key_tmpl = key_tmpl self.expire_in = expire_in def __call__(self, fn): @functools.wraps(fn) def _(*args, **kwargs): vv = inspect.getcallargs(fn, *args, **kwargs) key = self.key_tmpl.format(**vv) cached = mc.get(key) if cached is not None: return msgpack.loads(cached) rv = fn(*args, **kwargs) if rv is not None: mc.set(key, msgpack.dumps(rv), self.expire_in)
SCHEMA_FILE = 'schema.sql' BEANSDB_CFG = { 'localhost:11211': range(16), } MYSQL_HOST = 'localhost' MYSQL_PORT = 3306 MYSQL_USER = os.getenv('MYSQL_USER', 'root') MYSQL_PASSWORD = os.getenv('MYSQL_PASSWORD', 'root') MYSQL_DB = 'test_olo' MYSQL_CHARSET = 'utf8mb4' created_tables = False logger = logging.getLogger(__name__) mc = libmc.Client(['localhost:11211']) beansdb = BeansDBProxy(BEANSDB_CFG) mysql_conn = None approot = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..') dbdir = os.path.join(approot, 'databases') schema_path = os.path.join(dbdir, SCHEMA_FILE) def init_tables(): setup_mysql_conn() cur = mysql_conn.cursor() tables = get_all_tables(cur) if not tables: create_tables(cur)
import libmc mc = libmc.Client(['127.0.0.1:7900']) res = mc.set("foo", "bar") print(res) value = mc.get("foo") print(value)
#!/usr/bin/env python # -*- coding: utf-8 -*- import os import sys import json import random from sets import Set import libmc from libmc import ( MC_HASH_MD5, MC_POLL_TIMEOUT, MC_CONNECT_TIMEOUT, MC_RETRY_TIMEOUT ) mc = libmc.Client( ["127.0.0.1:11219"],comp_threshold=0, noreply=False, prefix=None,hash_fn=MC_HASH_MD5, failover=False ) mc.config(MC_POLL_TIMEOUT, 100) # 100 ms mc.config(MC_CONNECT_TIMEOUT, 300) # 300 ms mc.config(MC_RETRY_TIMEOUT, 5) # 5 s #input: [nike, running, shoes] , Dict #output: all rewrite query def query_rewriter_helper(query_terms, synonyms_dict): if (len(query_terms) == 0): return [] if (len(query_terms) == 1): if query_terms[0] not in synonyms_dict: return [query_terms[0]] else: return list(synonyms_dict[query_terms[0]]) prev = query_rewriter_helper(query_terms[:-1], synonyms_dict)
import gevent import time import gevent.monkey import slow_memcached_server gevent.monkey.patch_time() import greenify greenify.greenify() import libmc for so_path in libmc.DYNAMIC_LIBRARIES: assert greenify.patch_lib(so_path) mc = libmc.Client(["127.0.0.1:%d" % slow_memcached_server.PORT]) mc.config(libmc._client.MC_POLL_TIMEOUT, slow_memcached_server.BLOCKING_SECONDS * 1000 * 2) # ms stack = [] def mc_sleep(): print 'begin mc sleep' stack.append('mc_sleep_begin') assert mc.set('foo', 'bar') stack.append('mc_sleep_end') print 'end mc sleep' def singer(): i = 0 for i in range(6): i += 1 print '[%d] Oh, jingle bells, jingle bells, Jingle all the way.' % i
def get_mc(): return libmc.Client(MEMCACHED_HOSTS, **MEMCACHED_CONFIG)
def setUp(self): self.mc = libmc.Client([TEST_SERVER], comp_threshold=1024)
def setUp(self): self.mc = libmc.Client([TEST_SERVER], comp_threshold=1024) self.mc.set_behavior(libmc.BEHAVIOR_BINARY_PROTOCOL, 1)