Exemplo n.º 1
0
class SingleServerCase(unittest.TestCase):
    def setUp(self):
        self.mc = Client(["127.0.0.1:21211"])
        self.compressed_mc = Client(["127.0.0.1:21211"], comp_threshold=1024)
        self.noreply_mc = Client(["127.0.0.1:21211"], noreply=True)

    def test_attribute(self):
        "Test attributes are accessible from Python code"
        mc = self.mc
        assert hasattr(mc, 'comp_threshold')
        assert hasattr(mc, 'servers')

    def test_misc(self):
        mc = self.mc
        mc.get_multi(['foo', 'tuiche'])
        mc.delete('foo')
        mc.delete('tuiche')
        assert mc.get('foo') is None
        assert mc.get('tuiche') is None

        mc.set('foo', 'biu')
        mc.set('tuiche', 'bb')
        assert mc.get('foo') == 'biu'
        assert mc.get('tuiche') == 'bb'
        assert (mc.get_multi(['foo', 'tuiche']) == {
            'foo': 'biu',
            'tuiche': 'bb'
        })
        mc.set_multi({'foo': 1024, 'tuiche': '8964'})
        assert (mc.get_multi(['foo', 'tuiche']) == {
            'foo': 1024,
            'tuiche': '8964'
        })

    def test_delete(self):
        mc = self.mc
        assert mc.set('smmf', 0xCA909) is True
        assert mc.get('smmf') == 0xCA909
        assert mc.delete('smmf') is True
        assert mc.delete_multi(['smmf']) is True
        assert mc.set('smmf', 0xCA909) is True
        assert mc.delete_multi(['smmf']) is True
        assert mc.delete('smmf') is True
        assert mc.delete('smmf') is True

    def test_incr_decr(self):
        mc = self.mc
        mc.set('wazi', 99)
        assert mc.incr('wazi', 1) == 100
        assert mc.decr('wazi', 1) == 99
        mc.delete('wazi')
        assert mc.incr('wazi', 1) is None
        assert mc.decr('wazi', 1) is None

    def test_cas(self):
        mc = self.mc
        mc.delete('bilinda')
        k1 = 'bilinda'
        v1 = 'butchers'
        assert mc.gets(k1) is None
        mc.set(k1, v1)
        v1_, ck = mc.gets('bilinda')
        v2 = 'ding'
        assert v1 == v1_
        assert mc.cas(k1, v2, 0, ck) is True
        assert mc.cas(k1, v1, 0, ck) is not True
        v2_, ck = mc.gets('bilinda')
        assert v2 == v2_

    def test_large(self):
        mc = self.mc
        BUF_500KB = 'i' * (500 * 1000)
        BUF_1MB = 'i' * (1000 * 1000)
        key_500kb = 'test_500kb'
        key_1mb = 'test_1mb'
        mc.delete(key_500kb)
        mc.delete(key_1mb)
        mc.set(key_500kb, BUF_500KB)
        assert mc.get(key_500kb) == BUF_500KB
        mc.set(key_1mb, BUF_1MB)
        assert mc.get(key_1mb) == BUF_1MB
        mc.delete(key_500kb)
        mc.delete(key_1mb)
        dct = {key_500kb: BUF_500KB, key_1mb: BUF_1MB}
        mc.set_multi(dct)
        assert mc.get_multi(dct.keys()) == dct

    def test_extra_large(self):
        threshold = 1000000
        mc = self.mc
        key_xl = 'test_very_large'
        val = 'i' * threshold
        assert mc.set(key_xl, val)
        assert mc.get(key_xl) == val
        val += 'b'
        assert mc.set(key_xl, val)
        assert mc.get(key_xl) == val

        val = 'i' * (threshold + threshold)
        assert mc.set(key_xl, val)
        assert mc.get(key_xl) == val

        val += 'b'
        assert mc.set(key_xl, val)
        assert mc.get(key_xl) == val

    def test_noreply(self):
        mc = self.noreply_mc
        assert mc.set('foo', 'bar')
        assert mc.touch('foo', 30)
        v, ck = mc.gets('foo')
        assert mc.cas('foo', 'bar2', 0, ck)
        mc.delete('foo')
        assert mc.set('foo', 1024)
        assert mc.incr('foo', 1) is None
        assert mc.get('foo') == 1025
        assert mc.decr('foo', 3) is None
        assert mc.get('foo') == 1022

    def test_injection(self):
        # credit to The New Page of Injections Book:
        #   Memcached Injections @ blackhat2014 [pdf](http://t.cn/RP0J10Z)
        mc = self.mc
        assert mc.delete('injected')
        assert mc.set('a' * 250, 'biu')
        assert not mc.set('a' * 251, 'biu')
        mc.set('a' * 251, 'set injected 0 3600 10\r\n1234567890')
        assert mc.get('injected') is None
        mc.delete('injected')
        mc.set('key1', '1234567890')
        mc.set(
            'key1 0', '123456789012345678901234567890\r\n'
            'set injected 0 3600 3\r\nINJ\r\n')
        assert mc.get('injected') is None

    def test_maxiov(self):
        key_tmpl = 'not_existed.%s'
        assert self.mc.get_multi([key_tmpl % i for i in range(10000)]) == {}

    def test_get_set_raw(self):
        self.mc.set('foo', 233)
        assert self.mc.get_raw('foo') == ('233', 2)
        self.mc.set_raw('foo', '2335', 0, 2)
        assert self.mc.get('foo') == 2335

    def test_stats(self):
        stats = self.mc.stats()
        for addr, dct in stats.iteritems():
            assert isinstance(dct['version'], str)
            assert (isinstance(dct['rusage_system'], float)
                    or isinstance(dct['rusage_user'], float))
            assert isinstance(dct['curr_connections'], int)

    def test_get_set_large_raw(self):
        key = 'large_raw_key'
        key_dup = '%s_dup' % key
        val = 'i' * int(_DOUBAN_CHUNK_SIZE * 1.5)
        for mc in (self.mc, self.compressed_mc):
            mc.set(key, val)
            assert mc.get(key) == val
            raw_val1, flags1 = mc.get_raw(key)
            assert mc.set_raw(key_dup, raw_val1, 0, flags1)
            assert mc.get_raw(key) == mc.get_raw(key_dup)

    def test_patch_no_compress(self):
        key = 'no_compress'
        val = '1024'
        self.mc.set(key, val, compress=False)
        large_patch = 'hahahaha' * 512
        self.mc.prepend(key, large_patch)
        assert self.mc.get(key) == large_patch + val
        self.mc.delete(key)

        self.mc.set(key, val, compress=False)
        large_patch = 'hahahaha' * 512
        self.mc.append(key, large_patch)
        assert self.mc.get(key) == val + large_patch
        self.mc.delete(key)
Exemplo n.º 2
0
class SingleServerCase(unittest.TestCase):
    def setUp(self):
        self.mc = Client(["127.0.0.1:21211"])
        self.compressed_mc = Client(["127.0.0.1:21211"], comp_threshold=1024)
        self.noreply_mc = Client(["127.0.0.1:21211"], noreply=True)

    def test_attribute(self):
        "Test attributes are accessible from Python code"
        mc = self.mc
        assert hasattr(mc, 'comp_threshold')
        assert hasattr(mc, 'servers')

    def test_misc(self):
        mc = self.mc
        mc.get_multi(['foo', 'tuiche'])
        mc.delete('foo')
        mc.delete('tuiche')
        assert mc.get('foo') is None
        assert mc.get('tuiche') is None

        mc.set('foo', 'biu')
        mc.set('tuiche', 'bb')
        assert mc.get('foo') == 'biu'
        assert mc.get('tuiche') == 'bb'
        assert (mc.get_multi(['foo', 'tuiche']) ==
                {'foo': 'biu', 'tuiche': 'bb'})
        mc.set_multi({'foo': 1024, 'tuiche': '8964'})
        assert (mc.get_multi(['foo', 'tuiche']) ==
                {'foo': 1024, 'tuiche': '8964'})

    def test_delete(self):
        mc = self.mc
        assert mc.set('smmf', 0xCA909) is True
        assert mc.get('smmf') == 0xCA909
        assert mc.delete('smmf') is True
        assert mc.delete_multi(['smmf']) is True
        assert mc.set('smmf', 0xCA909) is True
        assert mc.delete_multi(['smmf']) is True
        assert mc.delete('smmf') is True
        assert mc.delete('smmf') is True

    def test_incr_decr(self):
        mc = self.mc
        mc.set('wazi', 99)
        assert mc.incr('wazi', 1) == 100
        assert mc.decr('wazi', 1) == 99
        mc.delete('wazi')
        assert mc.incr('wazi', 1) is None
        assert mc.decr('wazi', 1) is None

    def test_cas(self):
        mc = self.mc
        mc.delete('bilinda')
        k1 = 'bilinda'
        v1 = 'butchers'
        assert mc.gets(k1) is None
        mc.set(k1, v1)
        v1_, ck = mc.gets('bilinda')
        v2 = 'ding'
        assert v1 == v1_
        assert mc.cas(k1, v2, 0, ck) is True
        assert mc.cas(k1, v1, 0, ck) is not True
        v2_, ck = mc.gets('bilinda')
        assert v2 == v2_

    def test_large(self):
        mc = self.mc
        BUF_500KB = 'i' * (500 * 1000)
        BUF_1MB = 'i' * (1000 * 1000)
        key_500kb = 'test_500kb'
        key_1mb = 'test_1mb'
        mc.delete(key_500kb)
        mc.delete(key_1mb)
        mc.set(key_500kb, BUF_500KB)
        assert mc.get(key_500kb) == BUF_500KB
        mc.set(key_1mb, BUF_1MB)
        assert mc.get(key_1mb) == BUF_1MB
        mc.delete(key_500kb)
        mc.delete(key_1mb)
        dct = {key_500kb: BUF_500KB, key_1mb: BUF_1MB}
        mc.set_multi(dct)
        assert mc.get_multi(dct.keys()) == dct

    def test_extra_large(self):
        threshold = 1000000
        mc = self.mc
        key_xl = 'test_very_large'
        val = 'i' * threshold
        assert mc.set(key_xl, val)
        assert mc.get(key_xl) == val
        val += 'b'
        assert mc.set(key_xl, val)
        assert mc.get(key_xl) == val

        val = 'i' * (threshold + threshold)
        assert mc.set(key_xl, val)
        assert mc.get(key_xl) == val

        val += 'b'
        assert mc.set(key_xl, val)
        assert mc.get(key_xl) == val

    def test_noreply(self):
        mc = self.noreply_mc
        assert mc.set('foo', 'bar')
        assert mc.touch('foo', 30)
        v, ck = mc.gets('foo')
        assert mc.cas('foo', 'bar2', 0, ck)
        mc.delete('foo')
        assert mc.set('foo', 1024)
        assert mc.incr('foo', 1) is None
        assert mc.get('foo') == 1025
        assert mc.decr('foo', 3) is None
        assert mc.get('foo') == 1022

    def test_injection(self):
        # credit to The New Page of Injections Book:
        #   Memcached Injections @ blackhat2014 [pdf](http://t.cn/RP0J10Z)
        mc = self.mc
        assert mc.delete('injected')
        assert mc.set('a' * 250, 'biu')
        assert not mc.set('a' * 251, 'biu')
        mc.set('a' * 251, 'set injected 0 3600 10\r\n1234567890')
        assert mc.get('injected') is None
        mc.delete('injected')
        mc.set('key1', '1234567890')
        mc.set('key1 0', '123456789012345678901234567890\r\n'
                         'set injected 0 3600 3\r\nINJ\r\n')
        assert mc.get('injected') is None

    def test_maxiov(self):
        key_tmpl = 'not_existed.%s'
        assert self.mc.get_multi([key_tmpl % i for i in range(10000)]) == {}

    def test_get_set_raw(self):
        self.mc.set('foo', 233)
        assert self.mc.get_raw('foo') == ('233', 2)
        self.mc.set_raw('foo', '2335', 0, 2)
        assert self.mc.get('foo') == 2335

    def test_stats(self):
        stats = self.mc.stats()
        for addr, dct in stats.iteritems():
            assert isinstance(dct['version'], str)
            assert (isinstance(dct['rusage_system'], float) or
                    isinstance(dct['rusage_user'], float))
            assert isinstance(dct['curr_connections'], int)

    def test_get_set_large_raw(self):
        key = 'large_raw_key'
        key_dup = '%s_dup' % key
        val = 'i' * int(_DOUBAN_CHUNK_SIZE * 1.5)
        for mc in (self.mc, self.compressed_mc):
            mc.set(key, val)
            assert mc.get(key) == val
            raw_val1, flags1 = mc.get_raw(key)
            assert mc.set_raw(key_dup, raw_val1, 0, flags1)
            assert mc.get_raw(key) == mc.get_raw(key_dup)

    def test_patch_no_compress(self):
        key = 'no_compress'
        val = '1024'
        self.mc.set(key, val, compress=False)
        large_patch = 'hahahaha' * 512
        self.mc.prepend(key, large_patch)
        assert self.mc.get(key) == large_patch + val
        self.mc.delete(key)

        self.mc.set(key, val, compress=False)
        large_patch = 'hahahaha' * 512
        self.mc.append(key, large_patch)
        assert self.mc.get(key) == val + large_patch
        self.mc.delete(key)
Exemplo n.º 3
0
class CmemcachedRegressionCase(unittest.TestCase):
    def setUp(self):
        host = "127.0.0.1"
        port = 21211
        self.server_addr = '%s:%d' % (host, port)
        self.mc = Client([self.server_addr], comp_threshold=1024)
        if cmemcached is not None:
            self.old_mc = cmemcached.Client([self.server_addr],
                                            comp_threshold=1024)

    def test_set_get(self):
        self.mc.set("key", "value")
        self.assertEqual(self.mc.get("key"), "value")

        self.mc.set("key_int", 1)
        self.assertEqual(self.mc.get("key_int"), True)

        self.mc.set("key_long", int(1234567890))
        self.assertEqual(self.mc.get("key_long"), int(1234567890))

        self.mc.set("key_object", BigObject())
        self.assertEqual(self.mc.get("key_object"), BigObject())

        big_object = BigObject('x', 1000001)
        self.mc.set("key_big_object", big_object)
        self.assertEqual(self.mc.get("key_big_object"), big_object)

    def test_set_get_none(self):
        self.assertEqual(self.mc.set('key', None), True)
        self.assertEqual(self.mc.get('key'), None)

    def test_chinese_set_get(self):
        key = '豆瓣'
        value = '在炎热的夏天我们无法停止上豆瓣'
        self.assertEqual(self.mc.set(key, value), True)

        self.assertEqual(self.mc.get(key), value)

    def test_unicode_set_get(self):
        key = "test_unicode_set_get"
        value = u"中文"
        self.assertEqual(self.mc.set(key, value), True)
        self.assertEqual(self.mc.get(key), value)

    def test_special_key(self):
        key = 'keke a kid'
        value = 1024
        self.assertEqual(self.mc.set(key, value), False)
        self.assertEqual(self.mc.get(key), None)
        key = 'u:keke a kid'
        self.assertEqual(self.mc.set(key, value), False)
        self.assertEqual(self.mc.get(key), None)

    def test_unicode_key(self):
        key1 = u"answer"
        key2 = u"答案"
        bytes_key1 = "answer"
        bytes_key2 = "答案"
        value = 42

        self.assertEqual(self.mc.set(key1, value), True)
        self.assertEqual(self.mc.get(key1), value)

        self.assertEqual(self.mc.set(key2, value), True)
        self.assertEqual(self.mc.get(key2), value)

        self.assertEqual(self.mc.incr(key2), value + 1)
        self.assertEqual(self.mc.get(key2), value + 1)

        self.assertEqual(self.mc.delete(key1), True)
        self.assertEqual(self.mc.get(key1), None)

        self.assertEqual(self.mc.add(key1, value), True)
        self.assertEqual(self.mc.get(key1), value)
        self.assertEqual(self.mc.add(key1, value), False)
        self.assertEqual(self.mc.set(key1, value), True)

        self.assertEqual(self.mc.get(bytes_key1), self.mc.get(key1))
        self.assertEqual(self.mc.get(bytes_key2), self.mc.get(key2))

    def test_add(self):
        key = 'test_add'
        self.mc.delete(key)
        self.assertEqual(self.mc.add(key, 'tt'), True)
        self.assertEqual(self.mc.get(key), 'tt')
        self.assertEqual(self.mc.add(key, 'tt'), 0)
        self.mc.delete(key + '2')
        self.assertEqual(self.mc.add(key + '2', range(10)), True)

    def test_replace(self):
        key = 'test_replace'
        self.mc.delete(key)
        self.assertEqual(self.mc.replace(key, ''), 0)
        self.assertEqual(self.mc.set(key, 'b'), True)
        self.assertEqual(self.mc.replace(key, 'a'), True)
        self.assertEqual(self.mc.get(key), 'a')

    def test_append(self):
        key = "test_append"
        value = b"append\n"
        self.mc.delete(key)
        self.assertEqual(self.mc.append(key, value), 0)
        self.mc.set(key, b"")
        self.assertEqual(self.mc.append(key, value), True)
        self.assertEqual(self.mc.append(key, value), True)
        self.assertEqual(self.mc.prepend(key, b'before\n'), True)
        self.assertEqual(self.mc.get(key), b'before\n' + value * 2)

    def test_set_multi(self):
        values = dict(
            ('key%s' % k, ('value%s' % k) * 100) for k in range(1000))
        values.update({' ': ''})
        self.assertEqual(self.mc.set_multi(values), False)
        del values[' ']
        self.assertEqual(self.mc.get_multi(list(values.keys())), values)

    def test_append_large(self):
        k = 'test_append_large'
        self.mc.set(k, b'a' * 2048)
        self.mc.append(k, b'bbbb')
        assert b'bbbb' not in self.mc.get(k)
        self.mc.set(k, b'a' * 2048, compress=False)
        self.mc.append(k, b'bbbb')
        assert b'bbbb' in self.mc.get(k)

    def test_incr(self):
        key = "Not_Exist"
        self.assertEqual(self.mc.incr(key), None)
        # key="incr:key1"
        # self.mc.set(key, "not_numerical")
        # self.assertEqual(self.mc.incr(key), 0)
        key = "incr:key2"
        self.mc.set(key, 2007)
        self.assertEqual(self.mc.incr(key), 2008)

    def test_decr(self):
        key = "Not_Exist"
        self.assertEqual(self.mc.decr(key), None)
        # key="decr:key1"
        # self.mc.set(key, "not_numerical")
        # self.assertEqual(self.mc.decr(key),0)
        key = "decr:key2"
        self.mc.set(key, 2009)
        self.assertEqual(self.mc.decr(key), 2008)

    def test_get_multi(self):
        keys = ["hello1", "hello2", "hello3"]
        values = ["vhello1", "vhello2", "vhello3"]
        for x in range(3):
            self.mc.set(keys[x], values[x])
            self.assertEqual(self.mc.get(keys[x]), values[x])
        result = self.mc.get_multi(keys)
        for x in range(3):
            self.assertEqual(result[keys[x]], values[x])

    def test_get_multi_invalid(self):
        keys = ["hello1", "hello2", "hello3"]
        values = ["vhello1", "vhello2", "vhello3"]
        for x in range(3):
            self.mc.set(keys[x], values[x])
            self.assertEqual(self.mc.get(keys[x]), values[x])
        invalid_keys = keys + ['hoho\r\n']
        result = self.mc.get_multi(invalid_keys)
        for x in range(3):
            self.assertEqual(result[keys[x]], values[x])
        result_new = self.mc.get_multi(keys)
        for x in range(3):
            self.assertEqual(result_new[keys[x]], values[x])

    def test_get_multi_big(self):
        keys = ["hello1", "hello2", "hello3"]
        values = [BigObject(str(i), 1000001) for i in range(3)]
        for x in range(3):
            self.mc.set(keys[x], values[x])
            self.assertEqual(self.mc.get(keys[x]), values[x])
        result = self.mc.get_multi(keys)
        for x in range(3):
            self.assertEqual(result[keys[x]], values[x])

    def test_get_multi_with_empty_string(self):
        keys = ["hello1", "hello2", "hello3"]
        for k in keys:
            self.mc.set(k, '')
        self.assertEqual(self.mc.get_multi(keys), dict(zip(keys, [""] * 3)))

    def test_bool(self):
        self.mc.set("bool", True)
        value = self.mc.get("bool")
        self.assertEqual(value, True)
        self.mc.set("bool_", False)
        value = self.mc.get("bool_")
        self.assertEqual(value, False)

    def testEmptyString(self):
        self.assertTrue(self.mc.set("str", ''))
        value = self.mc.get("str")
        self.assertEqual(value, '')

    def test_pickle(self):
        v = [{"v": BigObject('a', 10)}]
        self.mc.set("a", v)
        self.assertEqual(self.mc.get("a"), v)
        # TODO
        '''
        raw, flags = self.mc.get_raw("a")
        self.assertEqual(raw, pickle.dumps(v, -1))
        '''

    def test_no_pickle(self):
        v = NoPickle()
        self.assertEqual(self.mc.set("nopickle", v), None)
        self.assertEqual(self.mc.get("nopickle"), None)

    def test_marshal(self):
        v = [{2: {"a": 337}}]
        self.mc.set("a", v)
        self.assertEqual(self.mc.get("a"), v)
        # TODO
        '''
        raw, flags = self.mc.get_raw("a")
        self.assertEqual(raw, marshal.dumps(v, 2))
        '''

    def test_big_list(self):
        v = range(1024 * 1024)
        r = self.mc.set('big_list', v)

        self.assertEqual(r, True)
        self.assertEqual(self.mc.get('big_list'), v)

    def test_touch(self):
        self.mc.set('test', True)
        self.assertEqual(self.mc.get('test'), True)
        self.assertEqual(self.mc.touch('test', -1), True)
        self.assertEqual(self.mc.get('test'), None)

        self.mc.set('test', True)
        self.assertEqual(self.mc.get('test'), True)
        self.assertEqual(self.mc.touch('test', 1), True)
        time.sleep(1)
        self.assertEqual(self.mc.get('test'), None)

    def test_client_pickable(self):
        import pickle
        d = pickle.dumps(self.mc)
        self.mc = pickle.loads(d)
        self.test_stats()

    def test_stats(self):
        s = self.mc.stats()
        self.assertEqual(self.server_addr in s, True)
        st = s[self.server_addr]
        st_keys = {
            "pid",
            "uptime",
            "time",
            "version",
            "pointer_size",
            "rusage_user",
            "rusage_system",
            "curr_items",
            "total_items",
            "bytes",
            "curr_connections",
            "total_connections",
            "connection_structures",
            "cmd_get",
            "cmd_set",
            "get_hits",
            "get_misses",
            "evictions",
            "bytes_read",
            "bytes_written",
            "limit_maxbytes",
            "threads",
        }
        self.assertTrue(set(st.keys()) >= st_keys)
        ''' TODO
        mc = cmemcached.Client([INVALID_SERVER_ADDR, self.server_addr])
        s = mc.stats()
        self.assertEqual(len(s), 2)
        '''

    def test_append_multi(self):
        N = 10
        K = "test_append_multi_%d"
        data = b"after\n"
        for i in range(N):
            self.assertEqual(self.mc.set(K % i, b"before\n"), True)
        keys = [K % i for i in range(N)]
        # append
        self.assertEqual(self.mc.append_multi(keys, data), True)
        self.assertEqual(self.mc.get_multi(keys),
                         dict(zip(keys, [b"before\n" + data] * N)))
        # prepend
        self.assertEqual(self.mc.prepend_multi(keys, data), True)
        self.assertEqual(self.mc.get_multi(keys),
                         dict(zip(keys, [data + b"before\n" + data] * N)))
        # delete
        self.assertEqual(self.mc.delete_multi(keys), True)
        self.assertEqual(self.mc.get_multi(keys), {})

    def test_append_multi_performance(self):
        N = 70000
        K = "test_append_multi_%d"
        data = b"after\n"
        keys = [K % i for i in range(N)]
        t = time.time()
        self.mc.append_multi(keys, data)
        t = time.time() - t
        threshold = 5
        assert t < threshold, 'should append 7w key in %s secs, ' \
                              'actual val: %f' % (threshold, t)

    def test_get_host(self):
        host = self.mc.get_host_by_key("str")
        self.assertEqual(host, self.server_addr)

    def test_get_list(self):
        self.mc.set("a", 'a')
        self.mc.delete('b')
        v = self.mc.get_list(['a', 'b'])
        self.assertEqual(v, ['a', None])

    @pytest.mark.skipif(np is None or cmemcached is None,
                        reason='cmemcached and numpy are not installed')
    def test_general_get_set_regression(self):
        key = 'anykey'
        key_dup = '%s_dup' % key
        for val in ('x', np.uint32(1), np.int32(2), 0, int(0), False, True):
            self.old_mc.set(key, val)
            val2 = self.mc.get(key)
            assert val2 == val
            self.mc.set(key_dup, val)
            val3 = self.old_mc.get(key_dup)
            assert val3 == val

    @pytest.mark.skipif(cmemcached is None,
                        reason='cmemcached is not installed')
    def test_large_mc_split_regression(self):
        key = 'anykey'
        key_dup = '%s_dup' % key
        for val in ('i' * int(_DOUBAN_CHUNK_SIZE * 1.5), 'small_value'):
            self.old_mc.set(key, val)
            assert self.mc.get(key) == self.old_mc.get(key) == val
            self.mc.set(key, val)
            assert self.mc.get(key) == self.old_mc.get(key) == val

            raw_val1, flags1 = self.old_mc.get_raw(key)
            assert len(raw_val1) <= len(val)  # compressed
            assert self.old_mc.set_raw(key_dup, raw_val1, 0, flags1)
            assert self.old_mc.get_raw(key) == self.old_mc.get_raw(key_dup)

            raw_val2, flags2 = self.mc.get_raw(key)
            assert len(raw_val2) <= len(val)  # compressed
            assert self.mc.set_raw(key_dup, raw_val2, 0, flags2)
            assert self.mc.get_raw(key) == self.mc.get_raw(key_dup)
Exemplo n.º 4
0
class CmemcachedRegressionCase(unittest.TestCase):

    def setUp(self):
        host = "127.0.0.1"
        port = 21211
        self.server_addr = '%s:%d' % (host, port)
        self.mc = Client([self.server_addr], comp_threshold=1024)
        if cmemcached is not None:
            self.old_mc = cmemcached.Client([self.server_addr],
                                            comp_threshold=1024)

    def test_set_get(self):
        self.mc.set("key", "value")
        self.assertEqual(self.mc.get("key"), "value")

        self.mc.set("key_int", 1)
        self.assertEqual(self.mc.get("key_int"), True)

        self.mc.set("key_long", int(1234567890))
        self.assertEqual(self.mc.get("key_long"), int(1234567890))

        self.mc.set("key_object", BigObject())
        self.assertEqual(self.mc.get("key_object"), BigObject())

        big_object = BigObject('x', 1000001)
        self.mc.set("key_big_object", big_object)
        self.assertEqual(self.mc.get("key_big_object"), big_object)

    def test_set_get_none(self):
        self.assertEqual(self.mc.set('key', None), True)
        self.assertEqual(self.mc.get('key'), None)

    def test_chinese_set_get(self):
        key = '豆瓣'
        value = '在炎热的夏天我们无法停止上豆瓣'
        self.assertEqual(self.mc.set(key, value), True)

        self.assertEqual(self.mc.get(key), value)

    def test_unicode_set_get(self):
        key = "test_unicode_set_get"
        value = u"中文"
        self.assertEqual(self.mc.set(key, value), True)
        self.assertEqual(self.mc.get(key), value)

    def test_special_key(self):
        key = 'keke a kid'
        value = 1024
        self.assertEqual(self.mc.set(key, value), False)
        self.assertEqual(self.mc.get(key), None)
        key = 'u:keke a kid'
        self.assertEqual(self.mc.set(key, value), False)
        self.assertEqual(self.mc.get(key), None)

    def test_unicode_key(self):
        key1 = u"answer"
        key2 = u"答案"
        bytes_key1 = "answer"
        bytes_key2 = "答案"
        value = 42

        self.assertEqual(self.mc.set(key1, value), True)
        self.assertEqual(self.mc.get(key1), value)

        self.assertEqual(self.mc.set(key2, value), True)
        self.assertEqual(self.mc.get(key2), value)

        self.assertEqual(self.mc.incr(key2), value + 1)
        self.assertEqual(self.mc.get(key2), value + 1)

        self.assertEqual(self.mc.delete(key1), True)
        self.assertEqual(self.mc.get(key1), None)

        self.assertEqual(self.mc.add(key1, value), True)
        self.assertEqual(self.mc.get(key1), value)
        self.assertEqual(self.mc.add(key1, value), False)
        self.assertEqual(self.mc.set(key1, value), True)

        self.assertEqual(self.mc.get(bytes_key1), self.mc.get(key1))
        self.assertEqual(self.mc.get(bytes_key2), self.mc.get(key2))

    def test_add(self):
        key = 'test_add'
        self.mc.delete(key)
        self.assertEqual(self.mc.add(key, 'tt'), True)
        self.assertEqual(self.mc.get(key), 'tt')
        self.assertEqual(self.mc.add(key, 'tt'), 0)
        self.mc.delete(key + '2')
        self.assertEqual(self.mc.add(key + '2', range(10)), True)

    def test_replace(self):
        key = 'test_replace'
        self.mc.delete(key)
        self.assertEqual(self.mc.replace(key, ''), 0)
        self.assertEqual(self.mc.set(key, 'b'), True)
        self.assertEqual(self.mc.replace(key, 'a'), True)
        self.assertEqual(self.mc.get(key), 'a')

    def test_append(self):
        key = "test_append"
        value = b"append\n"
        self.mc.delete(key)
        self.assertEqual(self.mc.append(key, value), 0)
        self.mc.set(key, b"")
        self.assertEqual(self.mc.append(key, value), True)
        self.assertEqual(self.mc.append(key, value), True)
        self.assertEqual(self.mc.prepend(key, b'before\n'), True)
        self.assertEqual(self.mc.get(key), b'before\n' + value * 2)

    def test_set_multi(self):
        values = dict(('key%s' % k, ('value%s' % k) * 100)
                      for k in range(1000))
        values.update({' ': ''})
        self.assertEqual(self.mc.set_multi(values), False)
        del values[' ']
        self.assertEqual(self.mc.get_multi(list(values.keys())), values)

    def test_append_large(self):
        k = 'test_append_large'
        self.mc.set(k, b'a' * 2048)
        self.mc.append(k, b'bbbb')
        assert b'bbbb' not in self.mc.get(k)
        self.mc.set(k, b'a' * 2048, compress=False)
        self.mc.append(k, b'bbbb')
        assert b'bbbb' in self.mc.get(k)

    def test_incr(self):
        key = "Not_Exist"
        self.assertEqual(self.mc.incr(key), None)
        # key="incr:key1"
        # self.mc.set(key, "not_numerical")
        # self.assertEqual(self.mc.incr(key), 0)
        key = "incr:key2"
        self.mc.set(key, 2007)
        self.assertEqual(self.mc.incr(key), 2008)

    def test_decr(self):
        key = "Not_Exist"
        self.assertEqual(self.mc.decr(key), None)
        # key="decr:key1"
        # self.mc.set(key, "not_numerical")
        # self.assertEqual(self.mc.decr(key),0)
        key = "decr:key2"
        self.mc.set(key, 2009)
        self.assertEqual(self.mc.decr(key), 2008)

    def test_get_multi(self):
        keys = ["hello1", "hello2", "hello3"]
        values = ["vhello1", "vhello2", "vhello3"]
        for x in range(3):
            self.mc.set(keys[x], values[x])
            self.assertEqual(self.mc.get(keys[x]), values[x])
        result = self.mc.get_multi(keys)
        for x in range(3):
            self.assertEqual(result[keys[x]], values[x])

    def test_get_multi_invalid(self):
        keys = ["hello1", "hello2", "hello3"]
        values = ["vhello1", "vhello2", "vhello3"]
        for x in range(3):
            self.mc.set(keys[x], values[x])
            self.assertEqual(self.mc.get(keys[x]), values[x])
        invalid_keys = keys + ['hoho\r\n']
        result = self.mc.get_multi(invalid_keys)
        for x in range(3):
            self.assertEqual(result[keys[x]], values[x])
        result_new = self.mc.get_multi(keys)
        for x in range(3):
            self.assertEqual(result_new[keys[x]], values[x])

    def test_get_multi_big(self):
        keys = ["hello1", "hello2", "hello3"]
        values = [BigObject(str(i), 1000001) for i in range(3)]
        for x in range(3):
            self.mc.set(keys[x], values[x])
            self.assertEqual(self.mc.get(keys[x]), values[x])
        result = self.mc.get_multi(keys)
        for x in range(3):
            self.assertEqual(result[keys[x]], values[x])

    def test_get_multi_with_empty_string(self):
        keys = ["hello1", "hello2", "hello3"]
        for k in keys:
            self.mc.set(k, '')
        self.assertEqual(self.mc.get_multi(keys), dict(zip(keys, [""] * 3)))

    def test_bool(self):
        self.mc.set("bool", True)
        value = self.mc.get("bool")
        self.assertEqual(value, True)
        self.mc.set("bool_", False)
        value = self.mc.get("bool_")
        self.assertEqual(value, False)

    def testEmptyString(self):
        self.assertTrue(self.mc.set("str", ''))
        value = self.mc.get("str")
        self.assertEqual(value, '')

    def test_pickle(self):
        v = [{"v": BigObject('a', 10)}]
        self.mc.set("a", v)
        self.assertEqual(self.mc.get("a"), v)
        # TODO
        '''
        raw, flags = self.mc.get_raw("a")
        self.assertEqual(raw, pickle.dumps(v, -1))
        '''

    def test_no_pickle(self):
        v = NoPickle()
        with warnings.catch_warnings():
            warnings.simplefilter("ignore")
            self.assertEqual(self.mc.set("nopickle", v), None)
        self.assertEqual(self.mc.get("nopickle"), None)

    def test_marshal(self):
        v = [{2: {"a": 337}}]
        self.mc.set("a", v)
        self.assertEqual(self.mc.get("a"), v)
        # TODO
        '''
        raw, flags = self.mc.get_raw("a")
        self.assertEqual(raw, marshal.dumps(v, 2))
        '''

    def test_big_list(self):
        v = range(1024 * 1024)
        r = self.mc.set('big_list', v)

        self.assertEqual(r, True)
        self.assertEqual(self.mc.get('big_list'), v)

    def test_touch(self):
        self.mc.set('test', True)
        self.assertEqual(self.mc.get('test'), True)
        self.assertEqual(self.mc.touch('test', -1), True)
        self.assertEqual(self.mc.get('test'), None)

        self.mc.set('test', True)
        self.assertEqual(self.mc.get('test'), True)
        self.assertEqual(self.mc.touch('test', 1), True)
        time.sleep(1)
        self.assertEqual(self.mc.get('test'), None)

    def test_client_pickable(self):
        import pickle
        d = pickle.dumps(self.mc)
        self.mc = pickle.loads(d)
        self.test_stats()

    def test_stats(self):
        s = self.mc.stats()
        self.assertEqual(self.server_addr in s, True)
        st = s[self.server_addr]
        st_keys = {
            "pid",
            "uptime",
            "time",
            "version",
            "pointer_size",
            "rusage_user",
            "rusage_system",
            "curr_items",
            "total_items",
            "bytes",
            "curr_connections",
            "total_connections",
            "connection_structures",
            "cmd_get",
            "cmd_set",
            "get_hits",
            "get_misses",
            "evictions",
            "bytes_read",
            "bytes_written",
            "limit_maxbytes",
            "threads",
        }
        self.assertTrue(set(st.keys()) >= st_keys)
        ''' TODO
        mc = cmemcached.Client([INVALID_SERVER_ADDR, self.server_addr])
        s = mc.stats()
        self.assertEqual(len(s), 2)
        '''

    def test_append_multi(self):
        N = 10
        K = "test_append_multi_%d"
        data = b"after\n"
        for i in range(N):
            self.assertEqual(self.mc.set(K % i, b"before\n"), True)
        keys = [K % i for i in range(N)]
        # append
        self.assertEqual(self.mc.append_multi(keys, data), True)
        self.assertEqual(self.mc.get_multi(keys),
                         dict(zip(keys, [b"before\n" + data] * N)))
        # prepend
        self.assertEqual(self.mc.prepend_multi(keys, data), True)
        self.assertEqual(self.mc.get_multi(keys),
                         dict(zip(keys, [data + b"before\n" + data] * N)))
        # delete
        self.assertEqual(self.mc.delete_multi(keys), True)
        self.assertEqual(self.mc.get_multi(keys), {})

    def test_append_multi_performance(self):
        N = 70000
        K = "test_append_multi_%d"
        data = b"after\n"
        keys = [K % i for i in range(N)]
        t = time.time()
        self.mc.append_multi(keys, data)
        t = time.time() - t
        threshold = 5
        assert t < threshold, 'should append 7w key in %s secs, ' \
                              'actual val: %f' % (threshold, t)

    def test_get_host(self):
        host = self.mc.get_host_by_key("str")
        self.assertEqual(host, self.server_addr)

    def test_get_list(self):
        self.mc.set("a", 'a')
        self.mc.delete('b')
        v = self.mc.get_list(['a', 'b'])
        self.assertEqual(v, ['a', None])

    @pytest.mark.skipif(np is None or cmemcached is None,
                        reason='cmemcached and numpy are not installed')
    def test_general_get_set_regression(self):
        key = 'anykey'
        key_dup = '%s_dup' % key
        for val in ('x', np.uint32(1), np.int32(2), 0, int(0), False, True):
            self.old_mc.set(key, val)
            val2 = self.mc.get(key)
            assert val2 == val
            self.mc.set(key_dup, val)
            val3 = self.old_mc.get(key_dup)
            assert val3 == val

    @pytest.mark.skipif(cmemcached is None,
                        reason='cmemcached is not installed')
    def test_large_mc_split_regression(self):
        key = 'anykey'
        key_dup = '%s_dup' % key
        for val in ('i' * int(_DOUBAN_CHUNK_SIZE * 1.5), 'small_value'):
            self.old_mc.set(key, val)
            assert self.mc.get(key) == self.old_mc.get(key) == val
            self.mc.set(key, val)
            assert self.mc.get(key) == self.old_mc.get(key) == val

            raw_val1, flags1 = self.old_mc.get_raw(key)
            assert len(raw_val1) <= len(val)  # compressed
            assert self.old_mc.set_raw(key_dup, raw_val1, 0, flags1)
            assert self.old_mc.get_raw(key) == self.old_mc.get_raw(key_dup)

            raw_val2, flags2 = self.mc.get_raw(key)
            assert len(raw_val2) <= len(val)  # compressed
            assert self.mc.set_raw(key_dup, raw_val2, 0, flags2)
            assert self.mc.get_raw(key) == self.mc.get_raw(key_dup)
Exemplo n.º 5
0
class SingleServerCase(unittest.TestCase):
    def setUp(self):
        self.mc = Client(["127.0.0.1:21211"])
        self.mc_alt = Client(["127.0.0.1:21211"])
        self.compressed_mc = Client(["127.0.0.1:21211"], comp_threshold=1024)
        self.noreply_mc = Client(["127.0.0.1:21211"], noreply=True)

    def test_attribute(self):
        "Test attributes are accessible from Python code"
        mc = self.mc
        assert hasattr(mc, 'comp_threshold')
        assert hasattr(mc, 'servers')

    def test_misc(self):
        mc = self.mc
        mc.get_multi(['foo', 'tuiche'])
        mc.delete('foo')
        mc.delete('tuiche')
        assert mc.get('foo') is None
        assert mc.get('tuiche') is None

        mc.set('foo', 'biu')
        mc.set('tuiche', 'bb')
        assert mc.get('foo') == 'biu'
        assert mc.get('tuiche') == 'bb'
        assert (mc.get_multi(['foo', 'tuiche']) == {
            'foo': 'biu',
            'tuiche': 'bb'
        })
        mc.set_multi({'foo': 1024, 'tuiche': '8964'})
        assert (mc.get_multi(['foo', 'tuiche']) == {
            'foo': 1024,
            'tuiche': '8964'
        })

    def test_delete(self):
        mc = self.mc
        assert mc.set('smmf', 0xCA909) is True
        assert mc.get('smmf') == 0xCA909
        assert mc.delete('smmf') is True
        assert mc.delete_multi(['smmf']) is True
        assert mc.set('smmf', 0xCA909) is True
        assert mc.delete_multi(['smmf']) is True
        assert mc.delete('smmf') is True
        assert mc.delete('smmf') is True

    def test_incr_decr(self):
        mc = self.mc
        mc.set('wazi', 99)
        assert mc.incr('wazi', 1) == 100
        assert mc.decr('wazi', 1) == 99
        mc.delete('wazi')
        assert mc.incr('wazi', 1) is None
        assert mc.decr('wazi', 1) is None

    def test_cas(self):
        mc = self.mc
        mc.delete('bilinda')
        k1 = 'bilinda'
        v1 = 'butchers'
        assert mc.gets(k1) is None
        mc.set(k1, v1)
        v1_, ck = mc.gets('bilinda')
        v2 = 'ding'
        assert v1 == v1_
        assert mc.cas(k1, v2, 0, ck) is True
        assert mc.cas(k1, v1, 0, ck) is not True
        v2_, ck = mc.gets('bilinda')
        assert v2 == v2_

    def test_large(self):
        mc = self.mc
        BUF_500KB = 'i' * (500 * 1000)
        BUF_1MB = 'i' * (1000 * 1000)
        key_500kb = 'test_500kb'
        key_1mb = 'test_1mb'
        mc.delete(key_500kb)
        mc.delete(key_1mb)
        mc.set(key_500kb, BUF_500KB)
        assert mc.get(key_500kb) == BUF_500KB
        mc.set(key_1mb, BUF_1MB)
        assert mc.get(key_1mb) == BUF_1MB
        mc.delete(key_500kb)
        mc.delete(key_1mb)
        dct = {key_500kb: BUF_500KB, key_1mb: BUF_1MB}
        mc.set_multi(dct)
        assert mc.get_multi(list(dct.keys())) == dct

    def test_extra_large(self):
        threshold = 1000000
        mc = self.mc
        key_xl = 'test_very_large'
        val = 'i' * threshold
        assert mc.set(key_xl, val)
        assert mc.get(key_xl) == val
        val += 'b'
        assert mc.set(key_xl, val)
        assert mc.get(key_xl) == val

        val = 'i' * (threshold + threshold)
        assert mc.set(key_xl, val)
        assert mc.get(key_xl) == val

        val += 'b'
        assert mc.set(key_xl, val)
        assert mc.get(key_xl) == val

    def test_noreply(self):
        mc = self.noreply_mc
        assert mc.set('foo', 'bar')
        assert mc.touch('foo', 30)
        v, ck = mc.gets('foo')
        assert mc.cas('foo', 'bar2', 0, ck)
        mc.delete('foo')
        assert mc.set('foo', 1024)
        assert mc.incr('foo', 1) is None
        assert mc.get('foo') == 1025
        assert mc.decr('foo', 3) is None
        assert mc.get('foo') == 1022

    def test_injection(self):
        # credit to The New Page of Injections Book:
        #   Memcached Injections @ blackhat2014 [pdf](http://t.cn/RP0J10Z)
        mc = self.mc
        assert mc.delete('injected')
        assert mc.set('a' * 250, 'biu')
        assert not mc.set('a' * 251, 'biu')
        mc.set('a' * 251, 'set injected 0 3600 10\r\n1234567890')
        assert mc.get('injected') is None
        mc.delete('injected')
        mc.set('key1', '1234567890')
        mc.set(
            'key1 0', '123456789012345678901234567890\r\n'
            'set injected 0 3600 3\r\nINJ\r\n')
        assert mc.get('injected') is None

    def test_maxiov(self):
        key_tmpl = 'not_existed.%s'
        assert self.mc.get_multi([key_tmpl % i for i in range(10000)]) == {}

    def test_get_set_raw(self):
        self.mc.set('foo', 233)
        assert self.mc.get_raw('foo') == (b'233', 2)
        self.mc.set_raw('foo', b'2335', 0, 2)
        assert self.mc.get('foo') == 2335

    def test_stats(self):
        stats = self.mc.stats()
        for addr, dct in stats.items():
            assert isinstance(dct['version'], basestring)
            assert (isinstance(dct['rusage_system'], float)
                    or isinstance(dct['rusage_user'], float))
            assert isinstance(dct['curr_connections'], int)

    def test_get_set_large_raw(self):
        key = 'large_raw_key'
        key_dup = '%s_dup' % key
        val = 'i' * int(_DOUBAN_CHUNK_SIZE * 1.5)
        for mc in (self.mc, self.compressed_mc):
            mc.set(key, val)
            assert mc.get(key) == val
            raw_val1, flags1 = mc.get_raw(key)
            assert mc.set_raw(key_dup, raw_val1, 0, flags1)
            assert mc.get_raw(key) == mc.get_raw(key_dup)

    def test_patch_no_compress(self):
        key = 'no_compress'
        val = b'1024'
        self.mc.set(key, val, compress=False)
        large_patch = b'hahahaha' * 512
        self.mc.prepend(key, large_patch)
        assert self.mc.get_raw(key)[0] == large_patch + val
        self.mc.delete(key)

        self.mc.set(key, val, compress=False)
        large_patch = b'hahahaha' * 512
        self.mc.append(key, large_patch)
        assert self.mc.get_raw(key)[0] == val + large_patch
        self.mc.delete(key)

    def test_quit(self):
        assert self.mc.delete('all_is_well')
        assert self.mc.set('all_is_well', 'bingo')
        assert self.mc_alt.delete('all_is_well')
        assert self.mc_alt.set('all_is_well', 'bingo')
        assert self.mc.version()  # establish all conns
        assert self.mc_alt.version()  # establish all conns
        nc1 = self.mc.stats()[self.mc.servers[0]]['curr_connections']
        nc2 = self.mc_alt.stats()[self.mc.servers[0]]['curr_connections']
        assert nc1 == nc2
        assert self.mc.quit()
        max_wait = 3
        while nc1 - 1 != nc2 and max_wait > 0:
            nc2 = self.mc_alt.stats()[self.mc.servers[0]]['curr_connections']
            max_wait -= 1
            time.sleep(1)
        assert nc1 - 1 == nc2
        # back to life immediately
        assert self.mc.get('all_is_well') == 'bingo'

    def test_flush_all(self):
        keys = ["flush_all_check_%s" % i for i in range(1000)]
        value = "testing_flush_all"
        dict_to_set = {key: value for key in keys}
        self.mc.set_multi(dict_to_set)
        retrieved_dct = self.mc.get_multi(keys)
        assert retrieved_dct == dict_to_set

        with pytest.raises(RuntimeError, match=r".*toggle.*"):
            rtn = self.mc.flush_all()
        self.mc.toggle_flush_all_feature(True)
        rtn = self.mc.flush_all()
        assert isinstance(rtn, list)
        assert rtn == self.mc.servers
        assert {} == self.mc.get_multi(keys)

        self.mc.toggle_flush_all_feature(False)
        with pytest.raises(RuntimeError, match=r".*toggle.*"):
            rtn = self.mc.flush_all()