def append_prepend_test(host, port, socket_module):
client = Client((host, port), socket_module=socket_module)
client.flush_all()
result = client.append(b'key', b'value', noreply=False)
tools.assert_equal(result, False)
result = client.get(b'key')
tools.assert_equal(result, None)
result = client.set(b'key', b'value', noreply=False)
tools.assert_equal(result, True)
result = client.append(b'key', b'after', noreply=False)
tools.assert_equal(result, True)
result = client.get(b'key')
tools.assert_equal(result, b'valueafter')
result = client.prepend(b'key1', b'value', noreply=False)
tools.assert_equal(result, False)
result = client.get(b'key1')
tools.assert_equal(result, None)
result = client.prepend(b'key', b'before', noreply=False)
tools.assert_equal(result, True)
result = client.get(b'key')
tools.assert_equal(result, b'beforevalueafter')
def test_incr_decr(host, port, socket_module):
client = Client((host, port), socket_module=socket_module)
client.flush_all()
result = client.incr(b'key', 1, noreply=False)
assert result is None
result = client.set(b'key', b'0', noreply=False)
assert result is True
result = client.incr(b'key', 1, noreply=False)
assert result == 1
def _bad_int():
client.incr(b'key', b'foobar')
with pytest.raises(MemcacheClientError):
_bad_int()
result = client.decr(b'key1', 1, noreply=False)
assert result is None
result = client.decr(b'key', 1, noreply=False)
assert result == 0
result = client.get(b'key')
assert result == b'0'
def append_prepend_test(host, port):
client = Client((host, port))
client.flush_all()
result = client.append('key', 'value', noreply=False)
tools.assert_equal(result, 'NOT_STORED')
result = client.get('key')
tools.assert_equal(result, None)
result = client.set('key', 'value', noreply=False)
tools.assert_equal(result, 'STORED')
result = client.append('key', 'after', noreply=False)
tools.assert_equal(result, 'STORED')
result = client.get('key')
tools.assert_equal(result, 'valueafter')
result = client.prepend('key1', 'value', noreply=False)
tools.assert_equal(result, 'NOT_STORED')
result = client.get('key1')
tools.assert_equal(result, None)
result = client.prepend('key', 'before', noreply=False)
tools.assert_equal(result, 'STORED')
result = client.get('key')
tools.assert_equal(result, 'beforevalueafter')
def append_prepend_test(host, port, socket_module):
client = Client((host, port), socket_module=socket_module)
client.flush_all()
result = client.append(b'key', b'value', noreply=False)
tools.assert_equal(result, False)
result = client.get(b'key')
tools.assert_equal(result, None)
result = client.set(b'key', b'value', noreply=False)
tools.assert_equal(result, True)
result = client.append(b'key', b'after', noreply=False)
tools.assert_equal(result, True)
result = client.get(b'key')
tools.assert_equal(result, b'valueafter')
result = client.prepend(b'key1', b'value', noreply=False)
tools.assert_equal(result, False)
result = client.get(b'key1')
tools.assert_equal(result, None)
result = client.prepend(b'key', b'before', noreply=False)
tools.assert_equal(result, True)
result = client.get(b'key')
tools.assert_equal(result, b'beforevalueafter')
def test_append_prepend(host, port, socket_module):
client = Client((host, port), socket_module=socket_module)
client.flush_all()
result = client.append(b'key', b'value', noreply=False)
assert result is False
result = client.get(b'key')
assert result is None
result = client.set(b'key', b'value', noreply=False)
assert result is True
result = client.append(b'key', b'after', noreply=False)
assert result is True
result = client.get(b'key')
assert result == b'valueafter'
result = client.prepend(b'key1', b'value', noreply=False)
assert result is False
result = client.get(b'key1')
assert result is None
result = client.prepend(b'key', b'before', noreply=False)
assert result is True
result = client.get(b'key')
assert result == b'beforevalueafter'
def test_errors(host, port, socket_module):
client = Client((host, port), socket_module=socket_module)
client.flush_all()
def _key_with_ws():
client.set('key with spaces', 'value', noreply=False)
tools.assert_raises(MemcacheIllegalInputError, _key_with_ws)
def _key_too_long():
client.set('x' * 1024, 'value', noreply=False)
tools.assert_raises(MemcacheClientError, _key_too_long)
def _unicode_key_in_set():
client.set(u'\u0FFF', 'value', noreply=False)
tools.assert_raises(MemcacheClientError, _unicode_key_in_set)
def _unicode_key_in_get():
client.get(u'\u0FFF')
tools.assert_raises(MemcacheClientError, _unicode_key_in_get)
def _unicode_value_in_set():
client.set('key', u'\u0FFF', noreply=False)
tools.assert_raises(MemcacheClientError, _unicode_value_in_set)
def append_prepend_test(host, port):
client = Client((host, port))
client.flush_all()
result = client.append('key', 'value', noreply=False)
tools.assert_equal(result, False)
result = client.get('key')
tools.assert_equal(result, None)
result = client.set('key', 'value', noreply=False)
tools.assert_equal(result, True)
result = client.append('key', 'after', noreply=False)
tools.assert_equal(result, True)
result = client.get('key')
tools.assert_equal(result, 'valueafter')
result = client.prepend('key1', 'value', noreply=False)
tools.assert_equal(result, False)
result = client.get('key1')
tools.assert_equal(result, None)
result = client.prepend('key', 'before', noreply=False)
tools.assert_equal(result, True)
result = client.get('key')
tools.assert_equal(result, 'beforevalueafter')
def test_errors(host, port):
client = Client((host, port))
client.flush_all()
def _key_with_ws():
client.set('key with spaces', 'value', noreply=False)
tools.assert_raises(MemcacheIllegalInputError, _key_with_ws)
def _key_too_long():
client.set('x' * 1024, 'value', noreply=False)
tools.assert_raises(MemcacheClientError, _key_too_long)
def _unicode_key_in_set():
client.set(u'\u0FFF', 'value', noreply=False)
tools.assert_raises(MemcacheClientError, _unicode_key_in_set)
def _unicode_key_in_get():
client.get(u'\u0FFF')
tools.assert_raises(MemcacheClientError, _unicode_key_in_get)
def _unicode_value_in_set():
client.set('key', u'\u0FFF', noreply=False)
tools.assert_raises(MemcacheClientError, _unicode_value_in_set)
def test_errors(host, port, socket_module):
client = Client((host, port), socket_module=socket_module)
client.flush_all()
def _key_with_ws():
client.set(b'key with spaces', b'value', noreply=False)
with pytest.raises(MemcacheIllegalInputError):
_key_with_ws()
def _key_too_long():
client.set(b'x' * 1024, b'value', noreply=False)
with pytest.raises(MemcacheClientError):
_key_too_long()
def _unicode_key_in_set():
client.set(six.u('\u0FFF'), b'value', noreply=False)
with pytest.raises(MemcacheClientError):
_unicode_key_in_set()
def _unicode_key_in_get():
client.get(six.u('\u0FFF'))
with pytest.raises(MemcacheClientError):
_unicode_key_in_get()
def _unicode_value_in_set():
client.set(b'key', six.u('\u0FFF'), noreply=False)
with pytest.raises(MemcacheClientError):
_unicode_value_in_set()
def gets_test(host, port):
client = Client((host, port))
client.flush_all()
result = client.gets('key')
tools.assert_equal(result, (None, None))
result = client.set('key', 'value', noreply=False)
tools.assert_equal(result, True)
result = client.gets('key')
tools.assert_equal(result[0], 'value')
def gets_test(host, port, socket_module):
client = Client((host, port), socket_module=socket_module)
client.flush_all()
result = client.gets(b'key')
tools.assert_equal(result, (None, None))
result = client.set(b'key', b'value', noreply=False)
tools.assert_equal(result, True)
result = client.gets(b'key')
tools.assert_equal(result[0], b'value')
def test_gets(host, port, socket_module):
client = Client((host, port), socket_module=socket_module)
client.flush_all()
result = client.gets(b'key')
assert result == (None, None)
result = client.set(b'key', b'value', noreply=False)
assert result is True
result = client.gets(b'key')
assert result[0] == b'value'
def gets_test(host, port):
client = Client((host, port))
client.flush_all()
result = client.gets('key')
tools.assert_equal(result, (None, None))
result = client.set('key', 'value', noreply=False)
tools.assert_equal(result, 'STORED')
result = client.gets('key')
tools.assert_equal(result[0], 'value')
def gets_test(host, port, socket_module):
client = Client((host, port), socket_module=socket_module)
client.flush_all()
result = client.gets(b'key')
tools.assert_equal(result, (None, None))
result = client.set(b'key', b'value', noreply=False)
tools.assert_equal(result, True)
result = client.gets(b'key')
tools.assert_equal(result[0], b'value')
def test_delete(host, port, socket_module):
client = Client((host, port), socket_module=socket_module)
client.flush_all()
result = client.delete(b'key', noreply=False)
assert result is False
result = client.get(b'key')
assert result is None
result = client.set(b'key', b'value', noreply=False)
assert result is True
result = client.delete(b'key', noreply=False)
assert result is True
result = client.get(b'key')
assert result is None
def delete_test(host, port, socket_module):
client = Client((host, port), socket_module=socket_module)
client.flush_all()
result = client.delete('key', noreply=False)
tools.assert_equal(result, False)
result = client.get('key')
tools.assert_equal(result, None)
result = client.set('key', 'value', noreply=False)
tools.assert_equal(result, True)
result = client.delete('key', noreply=False)
tools.assert_equal(result, True)
result = client.get('key')
tools.assert_equal(result, None)
def delete_test(host, port):
client = Client((host, port))
client.flush_all()
result = client.delete('key', noreply=False)
tools.assert_equal(result, False)
result = client.get('key')
tools.assert_equal(result, None)
result = client.set('key', 'value', noreply=False)
tools.assert_equal(result, True)
result = client.delete('key', noreply=False)
tools.assert_equal(result, True)
result = client.get('key')
tools.assert_equal(result, None)
def delete_test(host, port):
client = Client((host, port))
client.flush_all()
result = client.delete('key', noreply=False)
tools.assert_equal(result, 'NOT_FOUND')
result = client.get('key')
tools.assert_equal(result, None)
result = client.set('key', 'value', noreply=False)
tools.assert_equal(result, 'STORED')
result = client.delete('key', noreply=False)
tools.assert_equal(result, 'DELETED')
result = client.get('key')
tools.assert_equal(result, None)
def test_serialization_deserialization(host, port):
def _ser(value):
return json.dumps(value), 1
def _des(value, flags):
if flags == 1:
return json.loads(value)
return value
client = Client((host, port), serializer=_ser, deserializer=_des)
client.flush_all()
value = {'a': 'b', 'c': ['d']}
client.set('key', value)
result = client.get('key')
tools.assert_equal(result, value)
def test_serialization_deserialization(host, port):
def _ser(key, value):
return json.dumps(value), 1
def _des(key, value, flags):
if flags == 1:
return json.loads(value)
return value
client = Client((host, port), serializer=_ser, deserializer=_des)
client.flush_all()
value = {'a': 'b', 'c': ['d']}
client.set('key', value)
result = client.get('key')
tools.assert_equal(result, value)
def get_set_test(host, port):
client = Client((host, port))
client.flush_all()
result = client.get('key')
tools.assert_equal(result, None)
client.set('key', 'value', noreply=False)
result = client.get('key')
tools.assert_equal(result, 'value')
client.set('key2', 'value2', noreply=True)
result = client.get('key2')
tools.assert_equal(result, 'value2')
result = client.get_many(['key', 'key2'])
tools.assert_equal(result, {'key': 'value', 'key2': 'value2'})
def test_serialization_deserialization(host, port, socket_module):
def _ser(key, value):
return json.dumps(value).encode('ascii'), 1
def _des(key, value, flags):
if flags == 1:
return json.loads(value.decode('ascii'))
return value
client = Client((host, port), serializer=_ser, deserializer=_des,
socket_module=socket_module)
client.flush_all()
value = {'a': 'b', 'c': ['d']}
client.set(b'key', value)
result = client.get(b'key')
assert result == value
def get_set_test(host, port, socket_module):
client = Client((host, port), socket_module=socket_module)
client.flush_all()
result = client.get('key')
tools.assert_equal(result, None)
client.set(b'key', b'value', noreply=False)
result = client.get(b'key')
tools.assert_equal(result, b'value')
client.set(b'key2', b'value2', noreply=True)
result = client.get(b'key2')
tools.assert_equal(result, b'value2')
result = client.get_many([b'key', b'key2'])
tools.assert_equal(result, {b'key': b'value', b'key2': b'value2'})
result = client.get_many([])
tools.assert_equal(result, {})
def get_set_test(host, port):
client = Client((host, port))
client.flush_all()
result = client.get('key')
tools.assert_equal(result, None)
client.set('key', 'value', noreply=False)
result = client.get('key')
tools.assert_equal(result, 'value')
client.set('key2', 'value2', noreply=True)
result = client.get('key2')
tools.assert_equal(result, 'value2')
result = client.get_many(['key', 'key2'])
tools.assert_equal(result, {'key': 'value', 'key2': 'value2'})
result = client.get_many([])
tools.assert_equal(result, {})
def test_get_set(host, port, socket_module):
client = Client((host, port), socket_module=socket_module)
client.flush_all()
result = client.get('key')
assert result is None
client.set(b'key', b'value', noreply=False)
result = client.get(b'key')
assert result == b'value'
client.set(b'key2', b'value2', noreply=True)
result = client.get(b'key2')
assert result == b'value2'
result = client.get_many([b'key', b'key2'])
assert result == {b'key': b'value', b'key2': b'value2'}
result = client.get_many([])
assert result == {}
def get_set_test(host, port, socket_module):
client = Client((host, port), socket_module=socket_module)
client.flush_all()
result = client.get('key')
tools.assert_equal(result, None)
client.set(b'key', b'value', noreply=False)
result = client.get(b'key')
tools.assert_equal(result, b'value')
client.set(b'key2', b'value2', noreply=True)
result = client.get(b'key2')
tools.assert_equal(result, b'value2')
result = client.get_many([b'key', b'key2'])
tools.assert_equal(result, {b'key': b'value', b'key2': b'value2'})
result = client.get_many([])
tools.assert_equal(result, {})
def test_cas(host, port, socket_module):
client = Client((host, port), socket_module=socket_module)
client.flush_all()
result = client.cas(b'key', b'value', b'1', noreply=False)
assert result is None
result = client.set(b'key', b'value', noreply=False)
assert result is True
result = client.cas(b'key', b'value', b'1', noreply=False)
assert result is False
result, cas = client.gets(b'key')
assert result == b'value'
result = client.cas(b'key', b'value1', cas, noreply=False)
assert result is True
result = client.cas(b'key', b'value2', cas, noreply=False)
assert result is False
def cas_test(host, port):
client = Client((host, port))
client.flush_all()
result = client.cas('key', 'value', '1', noreply=False)
tools.assert_equal(result, None)
result = client.set('key', 'value', noreply=False)
tools.assert_equal(result, True)
result = client.cas('key', 'value', '1', noreply=False)
tools.assert_equal(result, False)
result, cas = client.gets('key')
tools.assert_equal(result, 'value')
result = client.cas('key', 'value1', cas, noreply=False)
tools.assert_equal(result, True)
result = client.cas('key', 'value2', cas, noreply=False)
tools.assert_equal(result, False)
def cas_test(host, port, socket_module):
client = Client((host, port), socket_module=socket_module)
client.flush_all()
result = client.cas(b'key', b'value', b'1', noreply=False)
tools.assert_equal(result, None)
result = client.set(b'key', b'value', noreply=False)
tools.assert_equal(result, True)
result = client.cas(b'key', b'value', b'1', noreply=False)
tools.assert_equal(result, False)
result, cas = client.gets(b'key')
tools.assert_equal(result, b'value')
result = client.cas(b'key', b'value1', cas, noreply=False)
tools.assert_equal(result, True)
result = client.cas(b'key', b'value2', cas, noreply=False)
tools.assert_equal(result, False)
def cas_test(host, port):
client = Client((host, port))
client.flush_all()
result = client.cas('key', 'value', '1', noreply=False)
tools.assert_equal(result, 'NOT_FOUND')
result = client.set('key', 'value', noreply=False)
tools.assert_equal(result, 'STORED')
result = client.cas('key', 'value', '1', noreply=False)
tools.assert_equal(result, 'EXISTS')
result, cas = client.gets('key')
tools.assert_equal(result, 'value')
result = client.cas('key', 'value1', cas, noreply=False)
tools.assert_equal(result, 'STORED')
result = client.cas('key', 'value2', cas, noreply=False)
tools.assert_equal(result, 'EXISTS')
def cas_test(host, port):
client = Client((host, port))
client.flush_all()
result = client.cas('key', 'value', '1', noreply=False)
tools.assert_equal(result, None)
result = client.set('key', 'value', noreply=False)
tools.assert_equal(result, True)
result = client.cas('key', 'value', '1', noreply=False)
tools.assert_equal(result, False)
result, cas = client.gets('key')
tools.assert_equal(result, 'value')
result = client.cas('key', 'value1', cas, noreply=False)
tools.assert_equal(result, True)
result = client.cas('key', 'value2', cas, noreply=False)
tools.assert_equal(result, False)
def cas_test(host, port, socket_module):
client = Client((host, port), socket_module=socket_module)
client.flush_all()
result = client.cas(b'key', b'value', b'1', noreply=False)
tools.assert_equal(result, None)
result = client.set(b'key', b'value', noreply=False)
tools.assert_equal(result, True)
result = client.cas(b'key', b'value', b'1', noreply=False)
tools.assert_equal(result, False)
result, cas = client.gets(b'key')
tools.assert_equal(result, b'value')
result = client.cas(b'key', b'value1', cas, noreply=False)
tools.assert_equal(result, True)
result = client.cas(b'key', b'value2', cas, noreply=False)
tools.assert_equal(result, False)
def add_replace_test(host, port):
client = Client((host, port))
client.flush_all()
result = client.add('key', 'value', noreply=False)
tools.assert_equal(result, True)
result = client.get('key')
tools.assert_equal(result, 'value')
result = client.add('key', 'value2', noreply=False)
tools.assert_equal(result, False)
result = client.get('key')
tools.assert_equal(result, 'value')
result = client.replace('key1', 'value1', noreply=False)
tools.assert_equal(result, False)
result = client.get('key1')
tools.assert_equal(result, None)
result = client.replace('key', 'value2', noreply=False)
tools.assert_equal(result, True)
result = client.get('key')
tools.assert_equal(result, 'value2')
def add_replace_test(host, port):
client = Client((host, port))
client.flush_all()
result = client.add('key', 'value', noreply=False)
tools.assert_equal(result, 'STORED')
result = client.get('key')
tools.assert_equal(result, 'value')
result = client.add('key', 'value2', noreply=False)
tools.assert_equal(result, 'NOT_STORED')
result = client.get('key')
tools.assert_equal(result, 'value')
result = client.replace('key1', 'value1', noreply=False)
tools.assert_equal(result, 'NOT_STORED')
result = client.get('key1')
tools.assert_equal(result, None)
result = client.replace('key', 'value2', noreply=False)
tools.assert_equal(result, 'STORED')
result = client.get('key')
tools.assert_equal(result, 'value2')
def add_replace_test(host, port, socket_module):
client = Client((host, port), socket_module=socket_module)
client.flush_all()
result = client.add(b'key', b'value', noreply=False)
tools.assert_equal(result, True)
result = client.get(b'key')
tools.assert_equal(result, b'value')
result = client.add(b'key', b'value2', noreply=False)
tools.assert_equal(result, False)
result = client.get(b'key')
tools.assert_equal(result, b'value')
result = client.replace(b'key1', b'value1', noreply=False)
tools.assert_equal(result, False)
result = client.get(b'key1')
tools.assert_equal(result, None)
result = client.replace(b'key', b'value2', noreply=False)
tools.assert_equal(result, True)
result = client.get(b'key')
tools.assert_equal(result, b'value2')
def test_add_replace(host, port, socket_module):
client = Client((host, port), socket_module=socket_module)
client.flush_all()
result = client.add(b'key', b'value', noreply=False)
assert result is True
result = client.get(b'key')
assert result == b'value'
result = client.add(b'key', b'value2', noreply=False)
assert result is False
result = client.get(b'key')
assert result == b'value'
result = client.replace(b'key1', b'value1', noreply=False)
assert result is False
result = client.get(b'key1')
assert result is None
result = client.replace(b'key', b'value2', noreply=False)
assert result is True
result = client.get(b'key')
assert result == b'value2'
def add_replace_test(host, port, socket_module):
client = Client((host, port), socket_module=socket_module)
client.flush_all()
result = client.add(b'key', b'value', noreply=False)
tools.assert_equal(result, True)
result = client.get(b'key')
tools.assert_equal(result, b'value')
result = client.add(b'key', b'value2', noreply=False)
tools.assert_equal(result, False)
result = client.get(b'key')
tools.assert_equal(result, b'value')
result = client.replace(b'key1', b'value1', noreply=False)
tools.assert_equal(result, False)
result = client.get(b'key1')
tools.assert_equal(result, None)
result = client.replace(b'key', b'value2', noreply=False)
tools.assert_equal(result, True)
result = client.get(b'key')
tools.assert_equal(result, b'value2')
def incr_decr_test(host, port, socket_module):
client = Client((host, port), socket_module=socket_module)
client.flush_all()
result = client.incr(b'key', 1, noreply=False)
tools.assert_equal(result, None)
result = client.set(b'key', b'0', noreply=False)
tools.assert_equal(result, True)
result = client.incr(b'key', 1, noreply=False)
tools.assert_equal(result, 1)
def _bad_int():
client.incr(b'key', b'foobar')
tools.assert_raises(MemcacheClientError, _bad_int)
result = client.decr(b'key1', 1, noreply=False)
tools.assert_equal(result, None)
result = client.decr(b'key', 1, noreply=False)
tools.assert_equal(result, 0)
result = client.get(b'key')
tools.assert_equal(result, b'0')
def incr_decr_test(host, port):
client = Client((host, port))
client.flush_all()
result = client.incr('key', 1, noreply=False)
tools.assert_equal(result, 'NOT_FOUND')
result = client.set('key', '0', noreply=False)
tools.assert_equal(result, 'STORED')
result = client.incr('key', 1, noreply=False)
tools.assert_equal(result, 1)
def _bad_int():
client.incr('key', 'foobar')
tools.assert_raises(MemcacheClientError, _bad_int)
result = client.decr('key1', 1, noreply=False)
tools.assert_equal(result, 'NOT_FOUND')
result = client.decr('key', 1, noreply=False)
tools.assert_equal(result, 0)
result = client.get('key')
tools.assert_equal(result, '0')
def incr_decr_test(host, port, socket_module):
client = Client((host, port), socket_module=socket_module)
client.flush_all()
result = client.incr(b'key', 1, noreply=False)
tools.assert_equal(result, None)
result = client.set(b'key', b'0', noreply=False)
tools.assert_equal(result, True)
result = client.incr(b'key', 1, noreply=False)
tools.assert_equal(result, 1)
def _bad_int():
client.incr(b'key', b'foobar')
tools.assert_raises(MemcacheClientError, _bad_int)
result = client.decr(b'key1', 1, noreply=False)
tools.assert_equal(result, None)
result = client.decr(b'key', 1, noreply=False)
tools.assert_equal(result, 0)
result = client.get(b'key')
tools.assert_equal(result, b'0')
class PyMemcachedCache(BaseCache):
"""A cache client based on pymemcache. implemented by pure python and support noreply.
"""
def __init__(self, config):
BaseCache.__init__(self, config)
self._client = PyMemcachedClient((config['host'], config['port']),
serializer=python_memcache_serializer, deserializer=python_memcache_deserializer,
connect_timeout=_DEFAULT_SOCKET_TIMEOUT, timeout=_DEFAULT_SOCKET_TIMEOUT,
key_prefix=config.get('key_prefix', ''))
def get(self, key):
return self._client.get(key)
def delete(self, key, noreply=False):
self._client.delete(key, noreply)
return True
def get_many(self, keys):
return self._client.get_many(keys)
def set(self, key, value, timeout=None, noreply=False):
if timeout is None:
timeout = self.default_timeout
return self._client.set(key, value, timeout, noreply)
def add(self, key, value, timeout=None, noreply=False):
if timeout is None:
timeout = self.default_timeout
return self._client.add(key, value, timeout, noreply)
def set_many(self, data, timeout=None, noreply=False):
if timeout is None:
timeout = self.default_timeout
return self._client.set_many(data, timeout, noreply)
def delete_many(self, keys, noreply=False):
return self._client.delete_many(keys, noreply)
def clear(self):
return self._client.flush_all(noreply=False)
def incr(self, key, delta=1, noreply=False):
return self._client.incr(key, delta, noreply)
def decr(self, key, delta=1, noreply=False):
return self._client.decr(key, delta, noreply)
class PyMemcachedCache(BaseCache):
"""A cache client based on pymemcache. implemented by pure python and support noreply.
"""
def __init__(self, config):
BaseCache.__init__(self, config)
self._client = PyMemcachedClient((config['host'], config['port']),
serializer=python_memcache_serializer, deserializer=python_memcache_deserializer,
connect_timeout=_DEFAULT_SOCKET_TIMEOUT, timeout=_DEFAULT_SOCKET_TIMEOUT,
key_prefix=config.get('key_prefix', ''))
def get(self, key):
return self._client.get(key)
def delete(self, key, noreply=False):
self._client.delete(key, noreply)
return True
def get_many(self, keys):
return self._client.get_many(keys)
def set(self, key, value, timeout=None, noreply=False):
if timeout is None:
timeout = self.default_timeout
return self._client.set(key, value, timeout, noreply)
def add(self, key, value, timeout=None, noreply=False):
if timeout is None:
timeout = self.default_timeout
return self._client.add(key, value, timeout, noreply)
def set_many(self, data, timeout=None, noreply=False):
if timeout is None:
timeout = self.default_timeout
return self._client.set_many(data, timeout, noreply)
def delete_many(self, keys, noreply=False):
return self._client.delete_many(keys, noreply)
def clear(self):
return self._client.flush_all(noreply=False)
def incr(self, key, delta=1, noreply=False):
return self._client.incr(key, delta, noreply)
def decr(self, key, delta=1, noreply=False):
return self._client.decr(key, delta, noreply)
class MemcachedStore(object):
"""Caching implementation that uses Memcached as data storage.
:param host: String representation of hostname or IP of the memcached server
:param port: Port number (int) on which the memcached server is listening
:param connect_timeout: optional float, seconds to wait for a connection to
the memcached server. Defaults to "forever" (uses the underlying
default socket timeout, which can be very long)
:param timeout: optional float, seconds to wait for send or recv calls on
the socket connected to memcached. Defaults to "forever" (uses the
underlying default socket timeout, which can be very long).
:param no_delay: optional bool, set the TCP_NODELAY flag, which may help
with performance in some cases. Defaults to False.
:param ignore_exc: optional bool, True to cause the "get", "gets",
"get_many" and "gets_many" calls to treat any errors as cache
misses. Defaults to True. Ie. if the cache is failing use the
Stormpath API.
:param socket_module: socket module to use, e.g. gevent.socket. Defaults to
the standard library's socket module.
:param key_prefix: Prefix of key. You can use this as namespace. Defaults
to b''.
"""
DEFAULT_TTL = 5 * 60 # seconds
def __init__(self, host='localhost', port=11211,
connect_timeout=None, timeout=None,
no_delay=False, ignore_exc=True,
key_prefix=b'', socket_module=socket, ttl=DEFAULT_TTL):
self.ttl = ttl
try:
from pymemcache.client import Client as Memcache
except ImportError:
raise RuntimeError('Memcached support is not available. Run "pip install pymemcache".')
self.memcache = Memcache(
(host, port),
serializer=json_serializer,
deserializer=json_deserializer,
connect_timeout=connect_timeout,
timeout=timeout,
socket_module=socket_module,
no_delay=no_delay,
ignore_exc=ignore_exc,
key_prefix=key_prefix)
@memcache_error_handling
def __getitem__(self, key):
entry = self.memcache.get(key)
if entry is None:
return None
return CacheEntry.parse(entry)
@memcache_error_handling
def __setitem__(self, key, entry):
self.memcache.set(key, entry, expire=self.ttl)
@memcache_error_handling
def __delitem__(self, key):
self.memcache.delete(key)
@memcache_error_handling
def clear(self):
self.memcache.flush_all()
@memcache_error_handling
def __len__(self):
return self.memcache.stats()['curr_items']
class MemcachedStore(object):
"""Caching implementation that uses Memcached as data storage.
:param host: String representation of hostname or IP of the memcached server
:param port: Port number (int) on which the memcached server is listening
:param connect_timeout: optional float, seconds to wait for a connection to
the memcached server. Defaults to "forever" (uses the underlying
default socket timeout, which can be very long)
:param timeout: optional float, seconds to wait for send or recv calls on
the socket connected to memcached. Defaults to "forever" (uses the
underlying default socket timeout, which can be very long).
:param no_delay: optional bool, set the TCP_NODELAY flag, which may help
with performance in some cases. Defaults to False.
:param ignore_exc: optional bool, True to cause the "get", "gets",
"get_many" and "gets_many" calls to treat any errors as cache
misses. Defaults to True. Ie. if the cache is failing use the
Stormpath API.
:param socket_module: socket module to use, e.g. gevent.socket. Defaults to
the standard library's socket module.
:param key_prefix: Prefix of key. You can use this as namespace. Defaults
to b''.
"""
DEFAULT_TTL = 5 * 60 # seconds
def __init__(self,
host='localhost',
port=11211,
connect_timeout=None,
timeout=None,
no_delay=False,
ignore_exc=True,
key_prefix=b'',
socket_module=socket,
ttl=DEFAULT_TTL):
self.ttl = ttl
try:
from pymemcache.client import Client as Memcache
except ImportError:
raise RuntimeError(
'Memcached support is not available. Run "pip install pymemcache".'
)
self.memcache = Memcache((host, port),
serializer=json_serializer,
deserializer=json_deserializer,
connect_timeout=connect_timeout,
timeout=timeout,
socket_module=socket_module,
no_delay=no_delay,
ignore_exc=ignore_exc,
key_prefix=key_prefix)
@memcache_error_handling
def __getitem__(self, key):
entry = self.memcache.get(key)
if entry is None:
return None
return CacheEntry.parse(entry)
@memcache_error_handling
def __setitem__(self, key, entry):
self.memcache.set(key, entry, expire=self.ttl)
@memcache_error_handling
def __delitem__(self, key):
self.memcache.delete(key)
@memcache_error_handling
def clear(self):
self.memcache.flush_all()
@memcache_error_handling
def __len__(self):
return self.memcache.stats()['curr_items']
# coding=utf-8
#import memcache
from collections import defaultdict
import json
from distance_scoring.scoring_doc import scoring_distance
from pymemcache.client import Client
mc = Client(("127.0.0.1", 12122))
mc.flush_all()
#def save_model_phi(filepath):
# with open(filepath, "r") as file_obj:
# for i, line in enumerate(file_obj):
# for j, score in enumerate(line.split()):
# score = float(score)
# mc.set("phi%s_%s"%(i,j), score)
def save_model_nw(filepath):
nw_sum = defaultdict(int)
with open(filepath, "r") as file_obj:
word_topic_dict = defaultdict(dict)
for word_idx, line in enumerate(file_obj):
for topic_idx, count in enumerate(line.split()):
count = int(count)
nw_sum[topic_idx] += count
if count > 0:
word_topic_dict[word_idx][topic_idx] = count
for word_idx, topic_count_dict in word_topic_dict.iteritems():
mc.set("nw%s" % (word_idx), topic_count_dict)
for topic_idx, count in nw_sum.iteritems():
class PyMemcached(Base):
"""A cache client based on pymemcache. Implemented by pure python and support noreply
"""
def __init__(self, host, port, timeout=None, prefix=''):
Base.__init__(self, timeout)
self._client = Client(
(host, port),
serializer=python_memcache_serializer,
deserializer=python_memcache_deserializer,
connect_timeout=_DEFAULT_SOCKET_TIMEOUT,
timeout=timeout or _DEFAULT_TIMEOUT,
key_prefix=prefix,
)
def get(self, key):
"""Look up the `key` in cache and return the value of it.
:param key: the `key` to be looked up
:returns: the value if it exists and is readable, else ``None``.
TODO: support __get__
"""
return self._client.get(_to_native(key))
def delete(self, key, noreply=False):
"""Delete `key` from cache.
:param key: the `key` to delete.
:param noreply: instruct the server to not reply.
:returns: whether the key been deleted.
:rtype: boolean
TODO: __del__
"""
self._client.delete(_to_native(key), noreply)
return True
def get_values(self, *keys):
"""Get valeus by keys
foo, bar = cache.get_values('foo', 'bar')
Share same error handling with :meth:`get`
:param keys: the function acception multiple keys as positional arguments
"""
return self._client.get_many([_to_native(key) for key in keys])
def set(self, key, value, timeout=None, noreply=False):
"""Add a new key/value to the cache (overwrite value, if key exists)
:param key: the key to set
:param value: the value of the key
:param timeout: the cache timeout for the key.
If not specificed, use the default timeout.
If specified 0, key will never expire
:param noreply: instructs the server to not reply
:returns: Whether the key existed and has been set
:rtype: boolean
TODO: __set__
"""
if timeout is None:
timeout = self.timeout
return self._client.set(_to_native(key), value, timeout, noreply)
def set_not_overwrite(self, key, value, timeout=None, noreply=False):
"""Works like :meth:`set` but does not overwrite the existing value
:param key: the key to set
:param value: the value of the key
:param timeout: the cache timeout for the key.
If not specificed, use the default timeout.
If specified 0, key will never expire
:param noreply: instructs the server to not reply
:returns: Whether the key existed and has been set
:rtype: boolean
"""
if timeout is None:
timeout = self.timeout
return self._client.add(_to_native(key), value, timeout, noreply)
def set_many(self, timeout=None, noreply=False, **kw):
"""Sets multiple key-value pair
:param timeout: the cache timeout for the key.
If not specificed, use the default timeout.
If specified 0, key will never expire
:param noreply: instructs the server to not reply
:returns: Whether all key-value pairs have been set
:rtype: boolean
"""
if timeout is None:
timeout = self.timeout
res = self._client.set_many(
dict((_to_native(k), v) for k, v in kw.items()), timeout, noreply)
if isinstance(res, list):
# pymemcache 2.x+ returns a list of keys that failed to update, instead of hard-coded boolean
return not res
return res
def delete_many(self, noreply=False, *keys):
"""Delete multiple keys at once.
:param keys: The function accept multiple keys as positional arguments
:param noreply: instructs the server not reply
:returns: Whether all given keys have been deleted
:rtype: boolen
"""
return self._client.delete_many([_to_native(key) for key in keys],
noreply)
def incr(self, key, delta=1, noreply=False):
"""Increments the value of a key by `delta`. If the key does not yet exists it is initialized with `delta`
For supporting caches this is an atomic operation
:param key: the key to increment
:param delta: the delta to add
:param noreply: instructs the server not reply
:returns: The new value or ``None`` for backend errors.
"""
return self._client.incr(_to_native(key), delta, noreply)
def decr(self, key, delta=1, noreply=False):
"""Decrements the value of a key by `delta`. If the key does not yet exists it is initialized with `-delta`.
For supporting caches this is an atomic operation.
:param key: the key to increment
:param delta: the delta to subtruct
:param noreply: instructs the server not reply
:returns: The new value or `None` for backend errors.
"""
return self._client.decr(_to_native(key), delta, noreply)
def clear(self):
"""Clears the cache. Not all caches support completely clearing the cache
:returns: Whether the cache been cleared.
:rtype: boolean
"""
return self._client.flush_all(noreply=False)
def expire(self, key, timeout):
"""Set a timeout on key. After the timeout has expired, the key will automatically be deleted.
:param key: key to set timeout
:param timeout: timeout value in seconds
:returns: True if timeout was set, False if key does not exist or timeout could not be set
:rtype: boolean
"""
# This requires the version of memcached server >= 1.4.8
return self._client.touch(_to_native(key), timeout)
def invalidate_all():
if is_active:
client = Client((host, port))
client.flush_all()
client.quit()
def test_flush_all():
client = Client(None)
client.sock = MockSocket(['OK\r\n'])
result = client.flush_all(noreply=False)
tools.assert_equal(result, True)
def misc_test(host, port):
client = Client((host, port))
client.flush_all()
def test_misc(host, port, socket_module):
client = Client((host, port), socket_module=socket_module)
client.flush_all()
