def get_multi(self, keys):
"""
Get multiple keys from server.
Since keys are converted to b'' when six.PY3 the keys need to be decoded back
into string . e.g key='test' is read as b'test' and then decoded back to 'test'
This encode/decode does not work when key is already a six.binary_type hence
this function remembers which keys were originally sent as str so that
it only decoded those keys back to string which were sent as string
:param keys: A list of keys to from server.
:type keys: Collection
:return: A dict with all requested keys.
:rtype: dict
"""
# pipeline N-1 getkq requests, followed by a regular getk to uncork the
# server
n = len(keys)
if n == 0:
return {}
msg = b''
for i, key in enumerate(keys):
keybytes = str_to_bytes(key)
command = self.COMMANDS['getk' if i == n - 1 else 'getkq']
msg += struct.pack(
self.HEADER_STRUCT + command['struct'] % (len(keybytes), ),
self.MAGIC['request'], command['command'], len(keybytes), 0, 0,
0, len(keybytes), 0, 0, keybytes)
self._send(msg)
d = {}
opcode = -1
while opcode != self.COMMANDS['getk']['command']:
(magic, opcode, keylen, extlen, datatype, status, bodylen, opaque,
cas, extra_content) = self._get_response()
if status == self.STATUS['success']:
flags, key, value = struct.unpack(
'!L%ds%ds' % (keylen, bodylen - keylen - 4), extra_content)
d[key] = self.deserialize(value, flags), cas
elif status == self.STATUS['server_disconnected']:
break
elif status != self.STATUS['key_not_found']:
raise MemcachedException(
'Code: %d Message: %s' % (status, extra_content), status)
ret = {}
for key in keys:
keybytes = str_to_bytes(key)
if keybytes in d:
ret[key] = d[keybytes]
return ret
def delete(self, key, cas=0):
"""
Delete a key/value from server. If key existed and was deleted, return True.
:param key: Key's name to be deleted
:type key: six.string_types
:param cas: If set, only delete the key if its CAS value matches.
:type cas: int
:return: True in case o success and False in case of failure.
:rtype: bool
"""
logger.debug('Deleting key %s', key)
self._send(struct.pack(self.HEADER_STRUCT +
self.COMMANDS['delete']['struct'] % len(key),
self.MAGIC['request'],
self.COMMANDS['delete']['command'],
len(key), 0, 0, 0, len(key), 0, cas, str_to_bytes(key)))
(magic, opcode, keylen, extlen, datatype, status, bodylen, opaque,
cas, extra_content) = self._get_response()
if status == self.STATUS['server_disconnected']:
return False
if status != self.STATUS['success'] and status not in (self.STATUS['key_not_found'], self.STATUS['key_exists']):
raise MemcachedException('Code: %d message: %s' % (status, extra_content), status)
logger.debug('Key deleted %s', key)
return status != self.STATUS['key_exists']
def _incr_decr(self, command, key, value, default, time):
"""
Function which increments and decrements.
:param key: Key's name
:type key: six.string_types
:param value: Number to be (de|in)cremented
:type value: int
:param default: Default value if key does not exist.
:type default: int
:param time: Time in seconds to expire key.
:type time: int
:return: Actual value of the key on server
:rtype: int
"""
time = time if time >= 0 else self.MAXIMUM_EXPIRE_TIME
self._send(struct.pack(self.HEADER_STRUCT +
self.COMMANDS[command]['struct'] % len(key),
self.MAGIC['request'],
self.COMMANDS[command]['command'],
len(key),
20, 0, 0, len(key) + 20, 0, 0, value,
default, time, str_to_bytes(key)))
(magic, opcode, keylen, extlen, datatype, status, bodylen, opaque,
cas, extra_content) = self._get_response()
if status not in (self.STATUS['success'], self.STATUS['server_disconnected']):
raise MemcachedException('Code: %d Message: %s' % (status, extra_content), status)
if status == self.STATUS['server_disconnected']:
return 0
return struct.unpack('!Q', extra_content)[0]
def _set_add_replace(self,
command,
key,
value,
time,
cas=0,
compress_level=-1):
"""
Function to set/add/replace commands.
:param key: Key's name
:type key: six.string_types
:param value: A value to be stored on server.
:type value: object
:param time: Time in seconds that your key will expire.
:type time: int
:param cas: The CAS value that must be matched for this operation to complete, or 0 for no CAS.
:type cas: int
:param compress_level: How much to compress.
0 = no compression, 1 = fastest, 9 = slowest but best,
-1 = default compression level.
:type compress_level: int
:return: True in case of success and False in case of failure
:rtype: bool
"""
time = time if time >= 0 else self.MAXIMUM_EXPIRE_TIME
logger.debug('Setting/adding/replacing key %s.', key)
flags, value = self.serialize(value, compress_level=compress_level)
logger.debug('Value bytes %s.', len(value))
if isinstance(value, text_type):
value = value.encode('utf8')
keybytes = str_to_bytes(key)
self._send(
struct.pack(
self.HEADER_STRUCT + self.COMMANDS[command]['struct'] %
(len(keybytes), len(value)), self.MAGIC['request'],
self.COMMANDS[command]['command'], len(keybytes), 8, 0, 0,
len(keybytes) + len(value) + 8, 0, cas, flags, time, keybytes,
value))
(magic, opcode, keylen, extlen, datatype, status, bodylen, opaque, cas,
extra_content) = self._get_response()
if status != self.STATUS['success']:
if status == self.STATUS['key_exists']:
return False
elif status == self.STATUS['key_not_found']:
return False
elif status == self.STATUS['server_disconnected']:
return False
raise MemcachedException(
'Code: %d Message: %s' % (status, extra_content), status)
return True
def stats(self, key=None):
"""
Return server stats.
:param key: Optional if you want status from a key.
:type key: six.string_types
:return: A dict with server stats
:rtype: dict
"""
# TODO: Stats with key is not working.
if key is not None:
if isinstance(key, text_type):
key = str_to_bytes(key)
keylen = len(key)
packed = struct.pack(
self.HEADER_STRUCT + '%ds' % keylen,
self.MAGIC['request'],
self.COMMANDS['stat']['command'],
keylen, 0, 0, 0, keylen, 0, 0, key)
else:
packed = struct.pack(
self.HEADER_STRUCT,
self.MAGIC['request'],
self.COMMANDS['stat']['command'],
0, 0, 0, 0, 0, 0, 0)
self._send(packed)
value = {}
while True:
response = self._get_response()
status = response[5]
if status == self.STATUS['server_disconnected']:
break
keylen = response[2]
bodylen = response[6]
if keylen == 0 and bodylen == 0:
break
extra_content = response[-1]
key = extra_content[:keylen]
body = extra_content[keylen:bodylen]
value[key.decode() if isinstance(key, bytes) else key] = body
return value
def get_multi(self, keys):
"""
Get multiple keys from server.
:param keys: A list of keys to from server.
:type keys: list
:return: A dict with all requested keys.
:rtype: dict
"""
# pipeline N-1 getkq requests, followed by a regular getk to uncork the
# server
keys, last = keys[:-1], keys[-1]
if six.PY2:
msg = ''
else:
msg = b''
msg = msg.join([
struct.pack(self.HEADER_STRUCT +
self.COMMANDS['getkq']['struct'] % (len(key)),
self.MAGIC['request'],
self.COMMANDS['getkq']['command'],
len(key), 0, 0, 0, len(key), 0, 0, str_to_bytes(key))
for key in keys])
msg += struct.pack(self.HEADER_STRUCT +
self.COMMANDS['getk']['struct'] % (len(last)),
self.MAGIC['request'],
self.COMMANDS['getk']['command'],
len(last), 0, 0, 0, len(last), 0, 0, last.encode())
self._send(msg)
d = {}
opcode = -1
while opcode != self.COMMANDS['getk']['command']:
(magic, opcode, keylen, extlen, datatype, status, bodylen, opaque,
cas, extra_content) = self._get_response()
if status == self.STATUS['success']:
flags, key, value = struct.unpack('!L%ds%ds' %
(keylen, bodylen - keylen - 4),
extra_content)
d[key.decode()] = self.deserialize(value, flags), cas
elif status == self.STATUS['server_disconnected']:
break
elif status != self.STATUS['key_not_found']:
raise MemcachedException('Code: %d Message: %s' % (status, extra_content))
return d
def _set_add_replace(self, command, key, value, time, cas=0, compress_level=-1):
"""
Function to set/add/replace commands.
:param key: Key's name
:type key: six.string_types
:param value: A value to be stored on server.
:type value: object
:param time: Time in seconds that your key will expire.
:type time: int
:param cas: The CAS value that must be matched for this operation to complete, or 0 for no CAS.
:type cas: int
:param compress_level: How much to compress.
0 = no compression, 1 = fastest, 9 = slowest but best,
-1 = default compression level.
:type compress_level: int
:return: True in case of success and False in case of failure
:rtype: bool
"""
time = time if time >= 0 else self.MAXIMUM_EXPIRE_TIME
logger.debug('Setting/adding/replacing key %s.', key)
flags, value = self.serialize(value, compress_level=compress_level)
logger.debug('Value bytes %s.', len(value))
if isinstance(value, text_type):
value = value.encode('utf8')
self._send(struct.pack(self.HEADER_STRUCT +
self.COMMANDS[command]['struct'] % (len(key), len(value)),
self.MAGIC['request'],
self.COMMANDS[command]['command'],
len(key), 8, 0, 0, len(key) + len(value) + 8, 0, cas, flags,
time, str_to_bytes(key), value))
(magic, opcode, keylen, extlen, datatype, status, bodylen, opaque,
cas, extra_content) = self._get_response()
if status != self.STATUS['success']:
if status == self.STATUS['key_exists']:
return False
elif status == self.STATUS['key_not_found']:
return False
elif status == self.STATUS['server_disconnected']:
return False
raise MemcachedException('Code: %d Message: %s' % (status, extra_content), status)
return True
def delete_multi(self, keys):
"""
Delete multiple keys from server in one command.
:param keys: A list of keys to be deleted
:type keys: list
:return: True in case of success and False in case of failure.
:rtype: bool
"""
logger.debug('Deleting keys %r', keys)
if six.PY2:
msg = ''
else:
msg = b''
for key in keys:
msg += struct.pack(
self.HEADER_STRUCT +
self.COMMANDS['delete']['struct'] % len(key),
self.MAGIC['request'],
self.COMMANDS['delete']['command'],
len(key), 0, 0, 0, len(key), 0, 0, str_to_bytes(key))
msg += struct.pack(
self.HEADER_STRUCT +
self.COMMANDS['noop']['struct'],
self.MAGIC['request'],
self.COMMANDS['noop']['command'],
0, 0, 0, 0, 0, 0, 0)
self._send(msg)
opcode = -1
retval = True
while opcode != self.COMMANDS['noop']['command']:
(magic, opcode, keylen, extlen, datatype, status, bodylen, opaque,
cas, extra_content) = self._get_response()
if status != self.STATUS['success']:
retval = False
if status == self.STATUS['server_disconnected']:
break
return retval
def get(self, key):
"""
Get a key and its CAS value from server. If the value isn't cached, return
(None, None).
:param key: Key's name
:type key: six.string_types
:return: Returns (value, cas).
:rtype: object
"""
logger.debug('Getting key %s', key)
keybytes = str_to_bytes(key)
data = struct.pack(
self.HEADER_STRUCT + self.COMMANDS['get']['struct'] %
(len(keybytes), ), self.MAGIC['request'],
self.COMMANDS['get']['command'], len(keybytes), 0, 0, 0,
len(keybytes), 0, 0, keybytes)
self._send(data)
(magic, opcode, keylen, extlen, datatype, status, bodylen, opaque, cas,
extra_content) = self._get_response()
logger.debug('Value Length: %d. Body length: %d. Data type: %d',
extlen, bodylen, datatype)
if status != self.STATUS['success']:
if status == self.STATUS['key_not_found']:
logger.debug('Key not found. Message: %s', extra_content)
return None, None
if status == self.STATUS['server_disconnected']:
return None, None
raise MemcachedException(
'Code: %d Message: %s' % (status, extra_content), status)
flags, value = struct.unpack('!L%ds' % (bodylen - 4, ), extra_content)
return self.deserialize(value, flags), cas
def get(self, key):
"""
Get a key and its CAS value from server. If the value isn't cached, return
(None, None).
:param key: Key's name
:type key: six.string_type
:return: Returns (value, cas).
:rtype: object
"""
logger.info('Getting key %s' % key)
data = struct.pack(self.HEADER_STRUCT +
self.COMMANDS['get']['struct'] % (len(key)),
self.MAGIC['request'],
self.COMMANDS['get']['command'],
len(key), 0, 0, 0, len(key), 0, 0, str_to_bytes(key))
self._send(data)
(magic, opcode, keylen, extlen, datatype, status, bodylen, opaque,
cas, extra_content) = self._get_response()
logger.debug('Value Length: %d. Body length: %d. Data type: %d' % (
extlen, bodylen, datatype))
if status != self.STATUS['success']:
if status == self.STATUS['key_not_found']:
logger.debug('Key not found. Message: %s'
% extra_content)
return None, None
if status == self.STATUS['server_disconnected']:
return None, None
raise MemcachedException('Code: %d Message: %s' % (status, extra_content))
flags, value = struct.unpack('!L%ds' % (bodylen - 4, ), extra_content)
return self.deserialize(value, flags), cas
def set_multi(self, mappings, time=100, compress_level=-1):
"""
Set multiple keys with its values on server.
If a key is a (key, cas) tuple, insert as if cas(key, value, cas) had
been called.
:param mappings: A dict with keys/values
:type mappings: dict
:param time: Time in seconds that your key will expire.
:type time: int
:param compress_level: How much to compress.
0 = no compression, 1 = fastest, 9 = slowest but best,
-1 = default compression level.
:type compress_level: int
:return: True
:rtype: bool
"""
mappings = mappings.items()
msg = []
for key, value in mappings:
if isinstance(key, tuple):
key, cas = key
else:
cas = None
if cas == 0:
# Like cas(), if the cas value is 0, treat it as compare-and-set against not
# existing.
command = 'addq'
else:
command = 'setq'
flags, value = self.serialize(value, compress_level=compress_level)
m = struct.pack(self.HEADER_STRUCT +
self.COMMANDS[command]['struct'] % (len(key), len(value)),
self.MAGIC['request'],
self.COMMANDS[command]['command'],
len(key),
8, 0, 0, len(key) + len(value) + 8, 0, cas or 0,
flags, time, str_to_bytes(key), value)
msg.append(m)
m = struct.pack(self.HEADER_STRUCT +
self.COMMANDS['noop']['struct'],
self.MAGIC['request'],
self.COMMANDS['noop']['command'],
0, 0, 0, 0, 0, 0, 0)
msg.append(m)
if six.PY2:
msg = ''.join(msg)
else:
msg = b''.join(msg)
self._send(msg)
opcode = -1
retval = True
while opcode != self.COMMANDS['noop']['command']:
(magic, opcode, keylen, extlen, datatype, status, bodylen, opaque,
cas, extra_content) = self._get_response()
if status != self.STATUS['success']:
retval = False
if status == self.STATUS['server_disconnected']:
break
return retval
def get_multi(self, keys):
"""
Get multiple keys from server.
Since keys are converted to b'' when six.PY3 the keys need to be decoded back
into string . e.g key='test' is read as b'test' and then decoded back to 'test'
This encode/decode does not work when key is already a six.binary_type hence
this function remembers which keys were originally sent as str so that
it only decoded those keys back to string which were sent as string
:param keys: A list of keys to from server.
:type keys: list
:return: A dict with all requested keys.
:rtype: dict
"""
# pipeline N-1 getkq requests, followed by a regular getk to uncork the
# server
o_keys = keys
keys, last = keys[:-1], str_to_bytes(keys[-1])
if six.PY2:
msg = ''
else:
msg = b''
msg = msg.join([
struct.pack(self.HEADER_STRUCT +
self.COMMANDS['getkq']['struct'] % (len(key)),
self.MAGIC['request'],
self.COMMANDS['getkq']['command'],
len(key), 0, 0, 0, len(key), 0, 0, str_to_bytes(key))
for key in keys])
msg += struct.pack(self.HEADER_STRUCT +
self.COMMANDS['getk']['struct'] % (len(last)),
self.MAGIC['request'],
self.COMMANDS['getk']['command'],
len(last), 0, 0, 0, len(last), 0, 0, last)
self._send(msg)
d = {}
opcode = -1
while opcode != self.COMMANDS['getk']['command']:
(magic, opcode, keylen, extlen, datatype, status, bodylen, opaque,
cas, extra_content) = self._get_response()
if status == self.STATUS['success']:
flags, key, value = struct.unpack('!L%ds%ds' %
(keylen, bodylen - keylen - 4),
extra_content)
if six.PY2:
d[key] = self.deserialize(value, flags), cas
else:
try:
decoded_key = key.decode()
except UnicodeDecodeError:
d[key] = self.deserialize(value, flags), cas
else:
if decoded_key in o_keys:
d[decoded_key] = self.deserialize(value, flags), cas
else:
d[key] = self.deserialize(value, flags), cas
elif status == self.STATUS['server_disconnected']:
break
elif status != self.STATUS['key_not_found']:
raise MemcachedException('Code: %d Message: %s' % (status, extra_content), status)
return d
def set_multi(self, mappings, time=100, compress_level=-1):
"""
Set multiple keys with its values on server.
If a key is a (key, cas) tuple, insert as if cas(key, value, cas) had
been called.
:param mappings: A dict with keys/values
:type mappings: dict
:param time: Time in seconds that your key will expire.
:type time: int
:param compress_level: How much to compress.
0 = no compression, 1 = fastest, 9 = slowest but best,
-1 = default compression level.
:type compress_level: int
:return: List of keys that failed to be set.
:rtype: list
"""
mappings = list(mappings.items())
msg = []
for opaque, (key, value) in enumerate(mappings):
if isinstance(key, tuple):
key, cas = key
else:
cas = None
if cas == 0:
# Like cas(), if the cas value is 0, treat it as compare-and-set against not
# existing.
command = 'addq'
else:
command = 'setq'
keybytes = str_to_bytes(key)
flags, value = self.serialize(value, compress_level=compress_level)
m = struct.pack(
self.HEADER_STRUCT + self.COMMANDS[command]['struct'] %
(len(keybytes), len(value)), self.MAGIC['request'],
self.COMMANDS[command]['command'], len(keybytes), 8, 0, 0,
len(keybytes) + len(value) + 8, opaque, cas or 0, flags, time,
keybytes, value)
msg.append(m)
m = struct.pack(self.HEADER_STRUCT + self.COMMANDS['noop']['struct'],
self.MAGIC['request'],
self.COMMANDS['noop']['command'], 0, 0, 0, 0, 0, 0, 0)
msg.append(m)
msg = b''.join(msg)
self._send(msg)
opcode = -1
failed = []
while opcode != self.COMMANDS['noop']['command']:
(magic, opcode, keylen, extlen, datatype, status, bodylen, opaque,
cas, extra_content) = self._get_response()
if status == self.STATUS['server_disconnected']:
# Assume that the entire operation failed.
return list(key for key, value in mappings)
if status != self.STATUS['success']:
key, value = mappings[opaque]
if isinstance(key, tuple):
failed.append((key[0], cas))
else:
failed.append(key)
return failed
