Ejemplo n.º 1
0
def test_logger_should_be_called_when_set():
    logger = Mock()
    client = cmemcached.Client([INVALID_SERVER_ADDR], logger=logger)
    client.get('test_key_with_logger')
    client.set('test_key_with_logger', 'test_value_with_logger')
    logger.assert_called_with("[cmemcached]memcached_get: server "
                              "127.0.0.1:12345 error: CONNECTION FAILURE\n")
    def test_stats(self):
        s = self.mc.stats()
        self.assertEqual(TEST_SERVER in s, True)
        st = s[TEST_SERVER]
        st_keys = sorted([
            "pid",
            "uptime",
            "time",
            "version",
            "pointer_size",
            "rusage_user",
            "rusage_system",
            "curr_items",
            "total_items",
            "bytes",
            "curr_connections",
            "total_connections",
            "connection_structures",
            "cmd_get",
            "cmd_set",
            "get_hits",
            "get_misses",
            "evictions",
            "bytes_read",
            "bytes_written",
            "limit_maxbytes",
            "threads",
        ])
        self.assertEqual(sorted(st.keys()), st_keys)

        mc = cmemcached.Client(["localhost:11999", TEST_SERVER])
        s = mc.stats()
        self.assertEqual(len(s), 2)
def main():
    mc = cmemcached.Client([
        "127.0.0.1:11211", "127.0.0.1:11212", "127.0.0.1:11213",
        "127.0.0.1:11214"
    ])

    num = 200
    keyss = []
    for i in xrange(4):
        k = i
        keyss.append(['key%d_%d' % (k, j) for j in xrange(num)])

    valuess = []
    for i in (1000, 10000, 100000, 200000):
        values = [BigObject(str(j % 10), i) for j in xrange(num)]
        valuess.append(values)

    def test_set_get(mc, pairs):
        counter = 0
        for key, value in pairs:
            mc.set(key, value)
            if (counter % 4 == 0):
                #assert mc.get(key) == value
                pass
            counter += 1

    from time import time
    t = time()
    for i in xrange(100):
        for k in xrange(4):
            pairs = zip(keyss[k], valuess[k])
            test_set_get(mc, pairs)
    print time() - t
def main():
    mc = cmemcached.Client([
        "127.0.0.1:11211", "127.0.0.1:11212", "127.0.0.1:11213",
        "127.0.0.1:11214"
    ])

    num = 10000
    keys = ["key%d" % k for k in xrange(num)]

    success_counter = 0
    failure_counter = 0

    def print_counter():
        print "success_counter is ", success_counter
        print "failure_counter is ", failure_counter
        assert failure_counter + success_counter == num
        print "mis rate is ", float(failure_counter) / num

    while True:
        for key in keys:
            mc.set(key, "aa")
        for key in keys:
            if mc.get(key) == "aa":
                success_counter += 1
            else:
                failure_counter += 1
        print_counter()
        success_counter = 0
        failure_counter = 0
Ejemplo n.º 5
0
 def setUp(self):
     host = "127.0.0.1"
     port = 21211
     self.server_addr = '%s:%d' % (host, port)
     self.mc = Client([self.server_addr], comp_threshold=1024)
     if cmemcached is not None:
         self.old_mc = cmemcached.Client([self.server_addr],
                                         comp_threshold=1024)
Ejemplo n.º 6
0
def connect(server, **kwargs):
    c = cmemcached.Client([server], do_split=0, **kwargs)
    c.set_behavior(cmemcached.BEHAVIOR_CONNECT_TIMEOUT, 100)  # 0.1 s
    c.set_behavior(cmemcached.BEHAVIOR_POLL_TIMEOUT, 5 * 1000)  # 5 s
    c.set_behavior(cmemcached.BEHAVIOR_SERVER_FAILURE_LIMIT, 4)
    c.set_behavior(cmemcached.BEHAVIOR_RETRY_TIMEOUT, 10)  # 10 s
    c.set_behavior(cmemcached.BEHAVIOR_SUPPORT_CAS, 0)
    return c
Ejemplo n.º 7
0
def create_mc(addr, **kwargs):
    client = cmemcached.Client(addr, comp_threshold=1024, logger=log, **kwargs)
    client.set_behavior(cmemcached.BEHAVIOR_CONNECT_TIMEOUT, 10)  # 0.01s
    client.set_behavior(cmemcached.BEHAVIOR_POLL_TIMEOUT, 300)  # 0.3s
    client.set_behavior(cmemcached.BEHAVIOR_RETRY_TIMEOUT, 5)  # 5 sec
    client.set_behavior(cmemcached.BEHAVIOR_SERVER_FAILURE_LIMIT,
                        2)  # not used in v1.0
    return client
Ejemplo n.º 8
0
 def test_set_multi(self):
     values = dict(('key%s'%k, ('value%s'%k)*100) for k in range(1000))
     values.update({' ':''})
     self.assertEqual(self.mc.set_multi(values), 1)
     del values[' ']
     self.assertEqual(self.mc.get_multi(values.keys()), values)
     mc = cmemcached.Client(['localhost:11999'], comp_threshold=1024)
     self.assertEqual(mc.set_multi(values), 0)
Ejemplo n.º 9
0
    def test_last_error(self):
        self.assertEqual(self.mc.set('testkey', 'hh'), True)
        self.assertEqual(self.mc.get('testkey'), 'hh')
        self.assertEqual(self.mc.get_last_error(), 0)

        self.mc = cmemcached.Client(['localhost:11999'], comp_threshold=1024)
        self.assertEqual(self.mc.set('testkey', 'hh'), False)
        self.assertEqual(self.mc.get('testkey'), None)
        self.assertNotEqual(self.mc.get_last_error(), 1)
Ejemplo n.º 10
0
    def __init__(self, servers, default_timeout=300, key_prefix=None):
        BaseCache.__init__(self, default_timeout)
        if isinstance(servers, (list, tuple)):
            is_cmemcached = is_cmemcache = is_pylibmc = False
            try:
                import cmemcached as memcache
                is_cmemcached = True
            except ImportError:
                try:
                    import cmemcache as memcache
                    is_cmemcache = True
                except ImportError:
                    try:
                        import memcache
                        is_cmemcache = False
                        is_pylibmc = False
                    except ImportError:
                        try:
                            import pylibmc as memcache
                            is_cmemcache = False
                            is_pylibmc = True
                        except ImportError:
                            raise RuntimeError('no memcache module found')

            if is_cmemcache:
                # cmemcache has a bug that debuglog is not defined for the
                # client.  Whenever pickle fails you get a weird
                # AttributeError.
                client = memcache.Client(map(str, servers))
                try:
                    client.debuglog = lambda *a: None
                except Exception:
                    pass
            elif is_pylibmc or is_cmemcached:
                client = memcache.Client(servers, False)
            else:
                client = memcache.Client(servers, False, HIGHEST_PROTOCOL)
        else:
            client = servers

        self._client = client
        self.key_prefix = key_prefix
Ejemplo n.º 11
0
def main():
    parameters = ['set', 'add_server', 'remove_server']
    parameters_string = '|'.join(parameters)
    p = optparse.OptionParser(usage=("%prog " + parameters_string))

    options, arguments = p.parse_args()
    if len(arguments) != 1 or arguments[0] not in parameters:
        p.error('Only one parameter(in %s) are supported!' % parameters_string)
    print arguments[0]

    if arguments[0] == 'remove_server':
        mc = cmemcached.Client(
            ["127.0.0.1:11211", "127.0.0.1:11212", "127.0.0.1:11213"])
    else:
        mc = cmemcached.Client([
            "127.0.0.1:11211", "127.0.0.1:11212", "127.0.0.1:11213",
            "127.0.0.1:11214"
        ])

    num_tests = 10000
    count = 0
    keys = ['helloworld%d' % i for i in xrange(num_tests)]
    if arguments[0] == 'set':
        for key in keys:
            mc.set(key, 'aa')
        for key in keys:
            assert mc.get(key) == 'aa'

    if arguments[0] == 'add_server':
        mc.add_server(["127.0.0.1:11215"])
        for key in keys:
            if mc.get(key) == 'aa':
                count += 1
        print "hit rate:", float(count) / num_tests

    if arguments[0] == 'remove_server':
        for key in keys:
            if mc.get(key) == 'aa':
                count += 1
        print "hit rate:", float(count) / num_tests
 def test_ketama(self):
     mc = cmemcached.Client(
         ['localhost', 'myhost:11211', '127.0.0.1:11212', 'myhost:11213'])
     rs = {
         'test:10000': 'localhost',
         'test:20000': '127.0.0.1:11212',
         'test:30000': '127.0.0.1:11212',
         'test:40000': '127.0.0.1:11212',
         'test:50000': '127.0.0.1:11212',
         'test:60000': 'myhost:11213',
         'test:70000': '127.0.0.1:11212',
         'test:80000': '127.0.0.1:11212',
         'test:90000': '127.0.0.1:11212',
     }
     for k in rs:
         self.assertEqual(mc.get_host_by_key(k), rs[k])
Ejemplo n.º 13
0
    def test_last_error(self):
        from cmemcached import RETURN_MEMCACHED_SUCCESS, RETURN_MEMCACHED_NOTFOUND
        self.assertEqual(self.mc.set('testkey', 'hh'), True)
        self.assertEqual(self.mc.get('testkey'), 'hh')
        self.assertEqual(self.mc.get_last_error(), RETURN_MEMCACHED_SUCCESS)
        self.assertEqual(self.mc.get('testkey1'), None)
        self.assertEqual(self.mc.get_last_error(), RETURN_MEMCACHED_SUCCESS)
        self.assertEqual(self.mc.get_multi(['testkey']), {'testkey': 'hh'})
        self.assertEqual(self.mc.get_last_error(), RETURN_MEMCACHED_SUCCESS)
        self.assertEqual(self.mc.get_multi(['testkey1']), {})
        self.assertEqual(self.mc.get_last_error(), RETURN_MEMCACHED_SUCCESS)

        self.mc = cmemcached.Client(["localhost:11999"], comp_threshold=1024)
        self.assertEqual(self.mc.set('testkey', 'hh'), False)
        self.assertEqual(self.mc.get('testkey'), None)
        self.assertNotEqual(self.mc.get_last_error(), RETURN_MEMCACHED_SUCCESS)
        self.assertEqual(self.mc.get_multi(['testkey']), {})
        self.assertNotEqual(self.mc.get_last_error(), RETURN_MEMCACHED_SUCCESS)
    def test_last_error(self):
        from cmemcached import RETURN_MEMCACHED_SUCCESS
        self.assertEqual(self.mc.set('testkey', 'hh'), True)
        self.assertEqual(self.mc.get('testkey'), 'hh')
        self.assertEqual(self.mc.get_last_error(), RETURN_MEMCACHED_SUCCESS)
        self.assertEqual(self.mc.get('testkey1'), None)
        self.assertEqual(self.mc.get_last_error(), RETURN_MEMCACHED_SUCCESS)
        self.assertEqual(self.mc.get_multi(['testkey']), {'testkey': 'hh'})
        self.assertEqual(self.mc.get_last_error(), RETURN_MEMCACHED_SUCCESS)
        self.assertEqual(self.mc.get_multi(['testkey1']), {})
        self.assertEqual(self.mc.get_last_error(), RETURN_MEMCACHED_SUCCESS)

        mc = cmemcached.Client([INVALID_SERVER_ADDR], comp_threshold=1024)
        self.assertEqual(mc.set('testkey', 'hh'), False)
        self.assertEqual(mc.get('testkey'), None)
        self.assertNotEqual(mc.get_last_error(), RETURN_MEMCACHED_SUCCESS)
        self.assertEqual(mc.get_multi(['testkey']), {})
        self.assertNotEqual(mc.get_last_error(), RETURN_MEMCACHED_SUCCESS)
 def test_ketama(self):
     mc = cmemcached.Client(
         ['localhost', 'myhost:11211', '127.0.0.1:11212', 'myhost:11213'])
     #for i in range(10):
     #    k = 'test:%d' % (i*10000)
     #    print """'%s': '%s',""" % (k, mc.get_host_by_key(k))
     rs = {
         'test:10000': 'localhost',
         'test:20000': '127.0.0.1:11212',
         'test:30000': '127.0.0.1:11212',
         'test:40000': '127.0.0.1:11212',
         'test:50000': '127.0.0.1:11212',
         'test:60000': 'myhost:11213',
         'test:70000': '127.0.0.1:11212',
         'test:80000': '127.0.0.1:11212',
         'test:90000': '127.0.0.1:11212',
     }
     for k in rs:
         self.assertEqual(mc.get_host_by_key(k), rs[k])
Ejemplo n.º 16
0
def test_set_multi_return_failure():
    client = cmemcached.Client([INVALID_SERVER_ADDR])
    v = dict([(str(i), "vv%s" % i) for i in range(10)])
    r, failures = client.set_multi(v, return_failure=True)
    assert not r
    assert failures == v.keys()
Ejemplo n.º 17
0
def test_delete_multi_return_failure():
    client = cmemcached.Client([INVALID_SERVER_ADDR])
    keys = [str(i) for i in range(10)]
    r, failures = client.delete_multi(keys, return_failure=True)
    assert not r
    assert failures == keys
Ejemplo n.º 18
0
def test_cmemcached():
    import cmemcached
    mc = cmemcached.Client(["127.0.0.1:11211"])
    for i in range(N):
        mc.set('piet%d' % i, 'klaas%d' % i)
 def setUp(self):
     self.mc = cmemcached.Client([TEST_UNIX_SOCKET], comp_threshold=1024)
Ejemplo n.º 20
0
import time
import string
import random
time.time()

start = time.time()

import cmemcached as memcache
mc = memcache.Client(['localhost:8402'], debug=0)

data = 'x' * 100

count = 1000000

for i in xrange(count + 1):
    key = "thing" + str(i)
    rc = mc.set(key, data)
    if rc == 0:
        mc = memcache.Client(['localhost:8402'], debug=0)  #re connect
        print 'reconn'
    if i % 10000 == 0:
        print 'inserting', i

end = time.time()

elapsed = end - start

print 'cost', str(elapsed)
print(count + 0.0) / elapsed, ' per second'
Ejemplo n.º 21
0
connection = Connection(host=MYSQL_HOST,
                        port=MYSQL_PORT,
                        user=MYSQL_USER,
                        passwd=MYSQL_PASSWD,
                        db=MYSQL_DB,
                        charset='utf8')

try:
    import sae.const
except ImportError:
    import cmemcached
    from config import MEMCACHED_ADDR
    kw = {}
    kw['comp_threshold'] = 4096
    mc = cmemcached.Client(MEMCACHED_ADDR)
else:
    import pylibmc
    mc = pylibmc.Client()

try:
    from sae.kvdb import KVClient
    kv = KVClient()
except ImportError:
    from kvstore import kv

from zorm_sae.mc_connection import init_mc
import zorm_sae.config

zorm_sae.config.mc = mc = init_mc(mc,
                                  disable_local_cached=DISABLE_LOCAL_CACHED)
Ejemplo n.º 22
0
 def __init__(self, mc_server=['127.0.0.1:12111']):
     self._mc_data_pool = cmemcached.Client(mc_server)
Ejemplo n.º 23
0
# -*- coding: utf-8 -*-

import cmemcached

servers = ["127.0.0.1:11508"]

mc = cmemcached.Client(servers)
Ejemplo n.º 24
0
import cmemcached
class BigObject(object):
    def __init__(self, letter='1', size=10000):
        self.object = letter * size

    def __eq__(self, other):
        return self.object == other.object

mc = cmemcached.Client(['127.0.0.1:11211', '127.0.0.1:11212', '127.0.0.1:11213', '127.0.0.1:11214'])

num = 200
keyss = []
for i in xrange(4):
    k = i
    keyss.append(['key%d_%d' % (k, j) for j in xrange(num)])


valuess = []
for i in (1000, 10000, 100000, 200000):
    values = [BigObject(str(j%10), i) for j in xrange(num)]
    valuess.append(values)

def test_set_get(mc, pairs):
    counter = 0
    for key, value in pairs:
        mc.set(key, value)
        if(counter%4 == 0):
            #assert mc.get(key) == value
            pass
        counter += 1
Ejemplo n.º 25
0
 def setUp(self):
     self.mc = cmemcached.Client([TEST_SERVER], comp_threshold=1024)
Ejemplo n.º 26
0
import time
import string
import random

time.time()

start = time.time()

import cmemcached as memcache

mc = memcache.Client(['localhost:8402'], debug=1)

data = 'x' * 100

count = 1000000

for i in xrange(count + 1):
    key = "thing" + str(i)
    rc = mc.get(key)
    #print key
    if i % 5000 == 0:
        print 'getting', i

end = time.time()

elapsed = end - start

print 'cost', str(elapsed)
print(count + 0.0) / elapsed, ' per second'
Ejemplo n.º 27
0
 def setUp(self):
     self.mc = cmemcached.Client([TEST_SERVER], comp_threshold=1024)
     self.mc.set_behavior(cmemcached.BEHAVIOR_BINARY_PROTOCOL, 1)
Ejemplo n.º 28
0
def test_sys_stderr_should_have_no_output_when_no_logger_is_set(memcached):
    mc = cmemcached.Client([memcached])
    with patch('sys.stderr') as mock_stderr:
        mc.get('test_key_with_no_logger')
        mc.set('test_key_with_no_logger', 'test_value_with_no_logger')
        assert not mock_stderr.write.called
        self.object = letter * size

    def __eq__(self, other):
        return self.object == other.object

parameters = ['set', 'add_server', 'remove_server']
parameters_string = '|'.join(parameters)
p = optparse.OptionParser(usage=("%prog " + parameters_string))

options, arguments = p.parse_args()
if len(arguments) != 1 or arguments[0] not in parameters:
    p.error('Only one parameter(in %s) are supported!' % parameters_string)
print arguments[0]

if arguments[0] == 'remove_server':
    mc =  cmemcached.Client(["127.0.0.1:11211","127.0.0.1:11212","127.0.0.1:11213"])
else:
    mc =  cmemcached.Client(["127.0.0.1:11211","127.0.0.1:11212","127.0.0.1:11213", "127.0.0.1:11214"])

num_tests = 10000
count = 0
keys = ['helloworld%d' % i for i in xrange(num_tests)]
if arguments[0] == 'set':
    for key in keys:
        mc.set(key , 'aa')
    for key in keys:
        assert mc.get(key) == 'aa'

if arguments[0] == 'add_server':
    mc.add_server(["127.0.0.1:11215"])
    for key in keys:
Ejemplo n.º 30
0
import cmemcached as memcached
import os
import time
basedir="/home/sunjoy/test/tstdb/"
os.system(basedir+"/tst.sh start")

time.sleep(2)
print 'begin set'
mc=memcached.Client(['localhost:8402'])
for i in xrange(1,10001):
	mc.set('thing'+str(i),'\r\nasdf\rsf\nsfsfd'*i)
print 'set over'

time.sleep(2)
print 'stop tstserver'
os.system(basedir+"/tst.sh stop")
time.sleep(2)
result=os.popen("grep header '../../log/stdout.log' | awk -F' ' '{print $5}'").read().split("\n")
#print result

for i in xrange(1,10001):
	#print i
	assert(int(result[i-1]),i)
print result[:10]