def delete_test(host, port):
client = Client((host, port))
client.flush_all()
result = client.delete('key', noreply=False)
tools.assert_equal(result, False)
result = client.get('key')
tools.assert_equal(result, None)
result = client.set('key', 'value', noreply=False)
tools.assert_equal(result, True)
result = client.delete('key', noreply=False)
tools.assert_equal(result, True)
result = client.get('key')
tools.assert_equal(result, None)
def delete_test(host, port, socket_module):
client = Client((host, port), socket_module=socket_module)
client.flush_all()
result = client.delete(b'key', noreply=False)
tools.assert_equal(result, False)
result = client.get(b'key')
tools.assert_equal(result, None)
result = client.set(b'key', b'value', noreply=False)
tools.assert_equal(result, True)
result = client.delete(b'key', noreply=False)
tools.assert_equal(result, True)
result = client.get(b'key')
tools.assert_equal(result, None)
def delete_test(host, port):
client = Client((host, port))
client.flush_all()
result = client.delete('key', noreply=False)
tools.assert_equal(result, 'NOT_FOUND')
result = client.get('key')
tools.assert_equal(result, None)
result = client.set('key', 'value', noreply=False)
tools.assert_equal(result, 'STORED')
result = client.delete('key', noreply=False)
tools.assert_equal(result, 'DELETED')
result = client.get('key')
tools.assert_equal(result, None)
def test_delete(host, port, socket_module):
client = Client((host, port), socket_module=socket_module)
client.flush_all()
result = client.delete(b'key', noreply=False)
assert result is False
result = client.get(b'key')
assert result is None
result = client.set(b'key', b'value', noreply=False)
assert result is True
result = client.delete(b'key', noreply=False)
assert result is True
result = client.get(b'key')
assert result is None
def delete_test(host, port, socket_module):
client = Client((host, port), socket_module=socket_module)
client.flush_all()
result = client.delete(b'key', noreply=False)
tools.assert_equal(result, False)
result = client.get(b'key')
tools.assert_equal(result, None)
result = client.set(b'key', b'value', noreply=False)
tools.assert_equal(result, True)
result = client.delete(b'key', noreply=False)
tools.assert_equal(result, True)
result = client.get(b'key')
tools.assert_equal(result, None)
def delete_test(host, port):
client = Client((host, port))
client.flush_all()
result = client.delete('key', noreply=False)
tools.assert_equal(result, False)
result = client.get('key')
tools.assert_equal(result, None)
result = client.set('key', 'value', noreply=False)
tools.assert_equal(result, True)
result = client.delete('key', noreply=False)
tools.assert_equal(result, True)
result = client.get('key')
tools.assert_equal(result, None)
class PyMemcachedCache(BaseCache):
"""A cache client based on pymemcache. implemented by pure python and support noreply.
"""
def __init__(self, config):
BaseCache.__init__(self, config)
self._client = PyMemcachedClient((config['host'], config['port']),
serializer=python_memcache_serializer, deserializer=python_memcache_deserializer,
connect_timeout=_DEFAULT_SOCKET_TIMEOUT, timeout=_DEFAULT_SOCKET_TIMEOUT,
key_prefix=config.get('key_prefix', ''))
def get(self, key):
return self._client.get(key)
def delete(self, key, noreply=False):
self._client.delete(key, noreply)
return True
def get_many(self, keys):
return self._client.get_many(keys)
def set(self, key, value, timeout=None, noreply=False):
if timeout is None:
timeout = self.default_timeout
return self._client.set(key, value, timeout, noreply)
def add(self, key, value, timeout=None, noreply=False):
if timeout is None:
timeout = self.default_timeout
return self._client.add(key, value, timeout, noreply)
def set_many(self, data, timeout=None, noreply=False):
if timeout is None:
timeout = self.default_timeout
return self._client.set_many(data, timeout, noreply)
def delete_many(self, keys, noreply=False):
return self._client.delete_many(keys, noreply)
def clear(self):
return self._client.flush_all(noreply=False)
def incr(self, key, delta=1, noreply=False):
return self._client.incr(key, delta, noreply)
def decr(self, key, delta=1, noreply=False):
return self._client.decr(key, delta, noreply)
class PyMemcachedCache(BaseCache):
"""A cache client based on pymemcache. implemented by pure python and support noreply.
"""
def __init__(self, config):
BaseCache.__init__(self, config)
self._client = PyMemcachedClient((config['host'], config['port']),
serializer=python_memcache_serializer, deserializer=python_memcache_deserializer,
connect_timeout=_DEFAULT_SOCKET_TIMEOUT, timeout=_DEFAULT_SOCKET_TIMEOUT,
key_prefix=config.get('key_prefix', ''))
def get(self, key):
return self._client.get(key)
def delete(self, key, noreply=False):
self._client.delete(key, noreply)
return True
def get_many(self, keys):
return self._client.get_many(keys)
def set(self, key, value, timeout=None, noreply=False):
if timeout is None:
timeout = self.default_timeout
return self._client.set(key, value, timeout, noreply)
def add(self, key, value, timeout=None, noreply=False):
if timeout is None:
timeout = self.default_timeout
return self._client.add(key, value, timeout, noreply)
def set_many(self, data, timeout=None, noreply=False):
if timeout is None:
timeout = self.default_timeout
return self._client.set_many(data, timeout, noreply)
def delete_many(self, keys, noreply=False):
return self._client.delete_many(keys, noreply)
def clear(self):
return self._client.flush_all(noreply=False)
def incr(self, key, delta=1, noreply=False):
return self._client.incr(key, delta, noreply)
def decr(self, key, delta=1, noreply=False):
return self._client.decr(key, delta, noreply)
class Cache:
def __init__(self):
self.__mem = Client(**CACHE_CONFIGS) # memcache init. here
self._tasks = 0
def __len__(self):
return self._tasks
def set(self, task, user_id):
try:
if self.__mem.set(str(user_id), task.json()):
self._tasks += 1
except (MemcacheIllegalInputError, MemcacheUnknownCommandError,
TypeError) as e:
logging.critical("%s occured with setting(%s: %s)" %
(str(e), str(user_id), str(task.json())))
def get(self, user_id):
try:
res = self.__mem.get(str(user_id))
except (MemcacheIllegalInputError, MemcacheUnknownCommandError,
TypeError) as e:
logging.critical("%s occured with getting %s" %
(str(e), str(user_id)))
return None
return res
def pop(self, user_id):
try:
res = self.__mem.get(str(user_id))
self.__mem.delete(str(user_id))
except (MemcacheIllegalInputError, MemcacheUnknownCommandError,
TypeError) as e:
logging.critical("%s occured with popping %s" %
(str(e), str(user_id)))
return None
return res
class MemcachedStore(object):
"""Caching implementation that uses Memcached as data storage.
:param host: String representation of hostname or IP of the memcached server
:param port: Port number (int) on which the memcached server is listening
:param connect_timeout: optional float, seconds to wait for a connection to
the memcached server. Defaults to "forever" (uses the underlying
default socket timeout, which can be very long)
:param timeout: optional float, seconds to wait for send or recv calls on
the socket connected to memcached. Defaults to "forever" (uses the
underlying default socket timeout, which can be very long).
:param no_delay: optional bool, set the TCP_NODELAY flag, which may help
with performance in some cases. Defaults to False.
:param ignore_exc: optional bool, True to cause the "get", "gets",
"get_many" and "gets_many" calls to treat any errors as cache
misses. Defaults to True. Ie. if the cache is failing use the
Stormpath API.
:param socket_module: socket module to use, e.g. gevent.socket. Defaults to
the standard library's socket module.
:param key_prefix: Prefix of key. You can use this as namespace. Defaults
to b''.
"""
DEFAULT_TTL = 5 * 60 # seconds
def __init__(self, host='localhost', port=11211,
connect_timeout=None, timeout=None,
no_delay=False, ignore_exc=True,
key_prefix=b'', socket_module=socket, ttl=DEFAULT_TTL):
self.ttl = ttl
try:
from pymemcache.client import Client as Memcache
except ImportError:
raise RuntimeError('Memcached support is not available. Run "pip install pymemcache".')
self.memcache = Memcache(
(host, port),
serializer=json_serializer,
deserializer=json_deserializer,
connect_timeout=connect_timeout,
timeout=timeout,
socket_module=socket_module,
no_delay=no_delay,
ignore_exc=ignore_exc,
key_prefix=key_prefix)
@memcache_error_handling
def __getitem__(self, key):
entry = self.memcache.get(key)
if entry is None:
return None
return CacheEntry.parse(entry)
@memcache_error_handling
def __setitem__(self, key, entry):
self.memcache.set(key, entry, expire=self.ttl)
@memcache_error_handling
def __delitem__(self, key):
self.memcache.delete(key)
@memcache_error_handling
def clear(self):
self.memcache.flush_all()
@memcache_error_handling
def __len__(self):
return self.memcache.stats()['curr_items']
class MemcachedStore(object):
"""Caching implementation that uses Memcached as data storage.
:param host: String representation of hostname or IP of the memcached server
:param port: Port number (int) on which the memcached server is listening
:param connect_timeout: optional float, seconds to wait for a connection to
the memcached server. Defaults to "forever" (uses the underlying
default socket timeout, which can be very long)
:param timeout: optional float, seconds to wait for send or recv calls on
the socket connected to memcached. Defaults to "forever" (uses the
underlying default socket timeout, which can be very long).
:param no_delay: optional bool, set the TCP_NODELAY flag, which may help
with performance in some cases. Defaults to False.
:param ignore_exc: optional bool, True to cause the "get", "gets",
"get_many" and "gets_many" calls to treat any errors as cache
misses. Defaults to True. Ie. if the cache is failing use the
Stormpath API.
:param socket_module: socket module to use, e.g. gevent.socket. Defaults to
the standard library's socket module.
:param key_prefix: Prefix of key. You can use this as namespace. Defaults
to b''.
"""
DEFAULT_TTL = 5 * 60 # seconds
def __init__(self,
host='localhost',
port=11211,
connect_timeout=None,
timeout=None,
no_delay=False,
ignore_exc=True,
key_prefix=b'',
socket_module=socket,
ttl=DEFAULT_TTL):
self.ttl = ttl
try:
from pymemcache.client import Client as Memcache
except ImportError:
raise RuntimeError(
'Memcached support is not available. Run "pip install pymemcache".'
)
self.memcache = Memcache((host, port),
serializer=json_serializer,
deserializer=json_deserializer,
connect_timeout=connect_timeout,
timeout=timeout,
socket_module=socket_module,
no_delay=no_delay,
ignore_exc=ignore_exc,
key_prefix=key_prefix)
@memcache_error_handling
def __getitem__(self, key):
entry = self.memcache.get(key)
if entry is None:
return None
return CacheEntry.parse(entry)
@memcache_error_handling
def __setitem__(self, key, entry):
self.memcache.set(key, entry, expire=self.ttl)
@memcache_error_handling
def __delitem__(self, key):
self.memcache.delete(key)
@memcache_error_handling
def clear(self):
self.memcache.flush_all()
@memcache_error_handling
def __len__(self):
return self.memcache.stats()['curr_items']
def test_delete_not_found():
client = Client(None)
client.sock = MockSocket(['NOT_FOUND\r\n'])
result = client.delete('key', noreply=False)
tools.assert_equal(result, False)
class Memcached(object):
"""docstring for Memcached
基础的 memcache 使用抽象类
"""
def __init__(self, ip, port):
super(Memcached, self).__init__()
# link = [str(ip) + ":" + str(port)]
self.mc = Client(
(ip, int(port)),
serializer=serialize_json,
deserializer=deserialize_json,
connect_timeout=2,
)
def bytesDictToStrDict(self, data):
new_data = {}
for k, v in data.items():
key = k
value = v
if isinstance(k, bytes):
key = k.decode()
if isinstance(v, bytes):
value = v.decode()
new_data[key] = value
return new_data
def stats(self):
return self.mc.stats()
def stats_str(self):
return (self.bytesDictToStrDict(self.mc.stats()))
def get(self, key):
data = self.mc.get(key.encode())
if data:
return data.decode()
return data
# else:
# return 'None Value'
def delete(self, key):
return self.mc.delete(key.encode())
def set(self, key, value, expire=600):
return self.mc.set(key, value, expire)
# data = bytes(value,encoding="utf-8")
# return self.mc.set(key,data,expire)
def show_stats(self, key):
return self.stats().get(key.encode())
def get_connections_sum(self):
return int(self.stats().get("curr_connections".encode()))
def get_mem_rate(self):
data = self.stats()
memsum = int(data.get("limit_maxbytes".encode()))
memused = int(data.get("bytes".encode()))
return round(memused / memsum * 100, 2)
def flush_all(self):
return self.mc.flush_all()
def invalidate(key):
if is_active:
key = __build_key(key)
client = Client((host,port))
client.delete(key)
client.quit()
def test_delete_noreply():
client = Client(None)
client.sock = MockSocket([])
result = client.delete('key', noreply=True)
tools.assert_equal(result, True)
def test_delete_found():
client = Client(None)
client.sock = MockSocket(['DELETED\r\n'])
result = client.delete('key', noreply=False)
tools.assert_equal(result, True)
def test_delete_not_found():
client = Client(None)
client.sock = MockSocket(['NOT_FOUND\r\n'])
result = client.delete('key', noreply=False)
tools.assert_equal(result, False)
def invalidate(key):
if is_active:
key = __build_key(key)
client = Client((host, port))
client.delete(key)
client.quit()
def test_delete_found():
client = Client(None)
client.sock = MockSocket(['DELETED\r\n'])
result = client.delete('key', noreply=False)
tools.assert_equal(result, 'DELETED')
class PyMemcached(Base):
"""A cache client based on pymemcache. Implemented by pure python and support noreply
"""
def __init__(self, host, port, timeout=None, prefix=''):
Base.__init__(self, timeout)
self._client = Client(
(host, port),
serializer=python_memcache_serializer,
deserializer=python_memcache_deserializer,
connect_timeout=_DEFAULT_SOCKET_TIMEOUT,
timeout=timeout or _DEFAULT_TIMEOUT,
key_prefix=prefix,
)
def get(self, key):
"""Look up the `key` in cache and return the value of it.
:param key: the `key` to be looked up
:returns: the value if it exists and is readable, else ``None``.
TODO: support __get__
"""
return self._client.get(_to_native(key))
def delete(self, key, noreply=False):
"""Delete `key` from cache.
:param key: the `key` to delete.
:param noreply: instruct the server to not reply.
:returns: whether the key been deleted.
:rtype: boolean
TODO: __del__
"""
self._client.delete(_to_native(key), noreply)
return True
def get_values(self, *keys):
"""Get valeus by keys
foo, bar = cache.get_values('foo', 'bar')
Share same error handling with :meth:`get`
:param keys: the function acception multiple keys as positional arguments
"""
return self._client.get_many([_to_native(key) for key in keys])
def set(self, key, value, timeout=None, noreply=False):
"""Add a new key/value to the cache (overwrite value, if key exists)
:param key: the key to set
:param value: the value of the key
:param timeout: the cache timeout for the key.
If not specificed, use the default timeout.
If specified 0, key will never expire
:param noreply: instructs the server to not reply
:returns: Whether the key existed and has been set
:rtype: boolean
TODO: __set__
"""
if timeout is None:
timeout = self.timeout
return self._client.set(_to_native(key), value, timeout, noreply)
def set_not_overwrite(self, key, value, timeout=None, noreply=False):
"""Works like :meth:`set` but does not overwrite the existing value
:param key: the key to set
:param value: the value of the key
:param timeout: the cache timeout for the key.
If not specificed, use the default timeout.
If specified 0, key will never expire
:param noreply: instructs the server to not reply
:returns: Whether the key existed and has been set
:rtype: boolean
"""
if timeout is None:
timeout = self.timeout
return self._client.add(_to_native(key), value, timeout, noreply)
def set_many(self, timeout=None, noreply=False, **kw):
"""Sets multiple key-value pair
:param timeout: the cache timeout for the key.
If not specificed, use the default timeout.
If specified 0, key will never expire
:param noreply: instructs the server to not reply
:returns: Whether all key-value pairs have been set
:rtype: boolean
"""
if timeout is None:
timeout = self.timeout
res = self._client.set_many(
dict((_to_native(k), v) for k, v in kw.items()), timeout, noreply)
if isinstance(res, list):
# pymemcache 2.x+ returns a list of keys that failed to update, instead of hard-coded boolean
return not res
return res
def delete_many(self, noreply=False, *keys):
"""Delete multiple keys at once.
:param keys: The function accept multiple keys as positional arguments
:param noreply: instructs the server not reply
:returns: Whether all given keys have been deleted
:rtype: boolen
"""
return self._client.delete_many([_to_native(key) for key in keys],
noreply)
def incr(self, key, delta=1, noreply=False):
"""Increments the value of a key by `delta`. If the key does not yet exists it is initialized with `delta`
For supporting caches this is an atomic operation
:param key: the key to increment
:param delta: the delta to add
:param noreply: instructs the server not reply
:returns: The new value or ``None`` for backend errors.
"""
return self._client.incr(_to_native(key), delta, noreply)
def decr(self, key, delta=1, noreply=False):
"""Decrements the value of a key by `delta`. If the key does not yet exists it is initialized with `-delta`.
For supporting caches this is an atomic operation.
:param key: the key to increment
:param delta: the delta to subtruct
:param noreply: instructs the server not reply
:returns: The new value or `None` for backend errors.
"""
return self._client.decr(_to_native(key), delta, noreply)
def clear(self):
"""Clears the cache. Not all caches support completely clearing the cache
:returns: Whether the cache been cleared.
:rtype: boolean
"""
return self._client.flush_all(noreply=False)
def expire(self, key, timeout):
"""Set a timeout on key. After the timeout has expired, the key will automatically be deleted.
:param key: key to set timeout
:param timeout: timeout value in seconds
:returns: True if timeout was set, False if key does not exist or timeout could not be set
:rtype: boolean
"""
# This requires the version of memcached server >= 1.4.8
return self._client.touch(_to_native(key), timeout)
def test_delete_noreply():
client = Client(None)
client.sock = MockSocket([])
result = client.delete('key', noreply=True)
tools.assert_equal(result, None)
