def get_multi(self, keys): """ Get multiple keys from server. Since keys are converted to b'' when six.PY3 the keys need to be decoded back into string . e.g key='test' is read as b'test' and then decoded back to 'test' This encode/decode does not work when key is already a six.binary_type hence this function remembers which keys were originally sent as str so that it only decoded those keys back to string which were sent as string :param keys: A list of keys to from server. :type keys: Collection :return: A dict with all requested keys. :rtype: dict """ # pipeline N-1 getkq requests, followed by a regular getk to uncork the # server n = len(keys) if n == 0: return {} msg = b'' for i, key in enumerate(keys): keybytes = str_to_bytes(key) command = self.COMMANDS['getk' if i == n - 1 else 'getkq'] msg += struct.pack( self.HEADER_STRUCT + command['struct'] % (len(keybytes), ), self.MAGIC['request'], command['command'], len(keybytes), 0, 0, 0, len(keybytes), 0, 0, keybytes) self._send(msg) d = {} opcode = -1 while opcode != self.COMMANDS['getk']['command']: (magic, opcode, keylen, extlen, datatype, status, bodylen, opaque, cas, extra_content) = self._get_response() if status == self.STATUS['success']: flags, key, value = struct.unpack( '!L%ds%ds' % (keylen, bodylen - keylen - 4), extra_content) d[key] = self.deserialize(value, flags), cas elif status == self.STATUS['server_disconnected']: break elif status != self.STATUS['key_not_found']: raise MemcachedException( 'Code: %d Message: %s' % (status, extra_content), status) ret = {} for key in keys: keybytes = str_to_bytes(key) if keybytes in d: ret[key] = d[keybytes] return ret
def delete(self, key, cas=0): """ Delete a key/value from server. If key existed and was deleted, return True. :param key: Key's name to be deleted :type key: six.string_types :param cas: If set, only delete the key if its CAS value matches. :type cas: int :return: True in case o success and False in case of failure. :rtype: bool """ logger.debug('Deleting key %s', key) self._send(struct.pack(self.HEADER_STRUCT + self.COMMANDS['delete']['struct'] % len(key), self.MAGIC['request'], self.COMMANDS['delete']['command'], len(key), 0, 0, 0, len(key), 0, cas, str_to_bytes(key))) (magic, opcode, keylen, extlen, datatype, status, bodylen, opaque, cas, extra_content) = self._get_response() if status == self.STATUS['server_disconnected']: return False if status != self.STATUS['success'] and status not in (self.STATUS['key_not_found'], self.STATUS['key_exists']): raise MemcachedException('Code: %d message: %s' % (status, extra_content), status) logger.debug('Key deleted %s', key) return status != self.STATUS['key_exists']
def _incr_decr(self, command, key, value, default, time): """ Function which increments and decrements. :param key: Key's name :type key: six.string_types :param value: Number to be (de|in)cremented :type value: int :param default: Default value if key does not exist. :type default: int :param time: Time in seconds to expire key. :type time: int :return: Actual value of the key on server :rtype: int """ time = time if time >= 0 else self.MAXIMUM_EXPIRE_TIME self._send(struct.pack(self.HEADER_STRUCT + self.COMMANDS[command]['struct'] % len(key), self.MAGIC['request'], self.COMMANDS[command]['command'], len(key), 20, 0, 0, len(key) + 20, 0, 0, value, default, time, str_to_bytes(key))) (magic, opcode, keylen, extlen, datatype, status, bodylen, opaque, cas, extra_content) = self._get_response() if status not in (self.STATUS['success'], self.STATUS['server_disconnected']): raise MemcachedException('Code: %d Message: %s' % (status, extra_content), status) if status == self.STATUS['server_disconnected']: return 0 return struct.unpack('!Q', extra_content)[0]
def _set_add_replace(self, command, key, value, time, cas=0, compress_level=-1): """ Function to set/add/replace commands. :param key: Key's name :type key: six.string_types :param value: A value to be stored on server. :type value: object :param time: Time in seconds that your key will expire. :type time: int :param cas: The CAS value that must be matched for this operation to complete, or 0 for no CAS. :type cas: int :param compress_level: How much to compress. 0 = no compression, 1 = fastest, 9 = slowest but best, -1 = default compression level. :type compress_level: int :return: True in case of success and False in case of failure :rtype: bool """ time = time if time >= 0 else self.MAXIMUM_EXPIRE_TIME logger.debug('Setting/adding/replacing key %s.', key) flags, value = self.serialize(value, compress_level=compress_level) logger.debug('Value bytes %s.', len(value)) if isinstance(value, text_type): value = value.encode('utf8') keybytes = str_to_bytes(key) self._send( struct.pack( self.HEADER_STRUCT + self.COMMANDS[command]['struct'] % (len(keybytes), len(value)), self.MAGIC['request'], self.COMMANDS[command]['command'], len(keybytes), 8, 0, 0, len(keybytes) + len(value) + 8, 0, cas, flags, time, keybytes, value)) (magic, opcode, keylen, extlen, datatype, status, bodylen, opaque, cas, extra_content) = self._get_response() if status != self.STATUS['success']: if status == self.STATUS['key_exists']: return False elif status == self.STATUS['key_not_found']: return False elif status == self.STATUS['server_disconnected']: return False raise MemcachedException( 'Code: %d Message: %s' % (status, extra_content), status) return True
def stats(self, key=None): """ Return server stats. :param key: Optional if you want status from a key. :type key: six.string_types :return: A dict with server stats :rtype: dict """ # TODO: Stats with key is not working. if key is not None: if isinstance(key, text_type): key = str_to_bytes(key) keylen = len(key) packed = struct.pack( self.HEADER_STRUCT + '%ds' % keylen, self.MAGIC['request'], self.COMMANDS['stat']['command'], keylen, 0, 0, 0, keylen, 0, 0, key) else: packed = struct.pack( self.HEADER_STRUCT, self.MAGIC['request'], self.COMMANDS['stat']['command'], 0, 0, 0, 0, 0, 0, 0) self._send(packed) value = {} while True: response = self._get_response() status = response[5] if status == self.STATUS['server_disconnected']: break keylen = response[2] bodylen = response[6] if keylen == 0 and bodylen == 0: break extra_content = response[-1] key = extra_content[:keylen] body = extra_content[keylen:bodylen] value[key.decode() if isinstance(key, bytes) else key] = body return value
def get_multi(self, keys): """ Get multiple keys from server. :param keys: A list of keys to from server. :type keys: list :return: A dict with all requested keys. :rtype: dict """ # pipeline N-1 getkq requests, followed by a regular getk to uncork the # server keys, last = keys[:-1], keys[-1] if six.PY2: msg = '' else: msg = b'' msg = msg.join([ struct.pack(self.HEADER_STRUCT + self.COMMANDS['getkq']['struct'] % (len(key)), self.MAGIC['request'], self.COMMANDS['getkq']['command'], len(key), 0, 0, 0, len(key), 0, 0, str_to_bytes(key)) for key in keys]) msg += struct.pack(self.HEADER_STRUCT + self.COMMANDS['getk']['struct'] % (len(last)), self.MAGIC['request'], self.COMMANDS['getk']['command'], len(last), 0, 0, 0, len(last), 0, 0, last.encode()) self._send(msg) d = {} opcode = -1 while opcode != self.COMMANDS['getk']['command']: (magic, opcode, keylen, extlen, datatype, status, bodylen, opaque, cas, extra_content) = self._get_response() if status == self.STATUS['success']: flags, key, value = struct.unpack('!L%ds%ds' % (keylen, bodylen - keylen - 4), extra_content) d[key.decode()] = self.deserialize(value, flags), cas elif status == self.STATUS['server_disconnected']: break elif status != self.STATUS['key_not_found']: raise MemcachedException('Code: %d Message: %s' % (status, extra_content)) return d
def _set_add_replace(self, command, key, value, time, cas=0, compress_level=-1): """ Function to set/add/replace commands. :param key: Key's name :type key: six.string_types :param value: A value to be stored on server. :type value: object :param time: Time in seconds that your key will expire. :type time: int :param cas: The CAS value that must be matched for this operation to complete, or 0 for no CAS. :type cas: int :param compress_level: How much to compress. 0 = no compression, 1 = fastest, 9 = slowest but best, -1 = default compression level. :type compress_level: int :return: True in case of success and False in case of failure :rtype: bool """ time = time if time >= 0 else self.MAXIMUM_EXPIRE_TIME logger.debug('Setting/adding/replacing key %s.', key) flags, value = self.serialize(value, compress_level=compress_level) logger.debug('Value bytes %s.', len(value)) if isinstance(value, text_type): value = value.encode('utf8') self._send(struct.pack(self.HEADER_STRUCT + self.COMMANDS[command]['struct'] % (len(key), len(value)), self.MAGIC['request'], self.COMMANDS[command]['command'], len(key), 8, 0, 0, len(key) + len(value) + 8, 0, cas, flags, time, str_to_bytes(key), value)) (magic, opcode, keylen, extlen, datatype, status, bodylen, opaque, cas, extra_content) = self._get_response() if status != self.STATUS['success']: if status == self.STATUS['key_exists']: return False elif status == self.STATUS['key_not_found']: return False elif status == self.STATUS['server_disconnected']: return False raise MemcachedException('Code: %d Message: %s' % (status, extra_content), status) return True
def delete_multi(self, keys): """ Delete multiple keys from server in one command. :param keys: A list of keys to be deleted :type keys: list :return: True in case of success and False in case of failure. :rtype: bool """ logger.debug('Deleting keys %r', keys) if six.PY2: msg = '' else: msg = b'' for key in keys: msg += struct.pack( self.HEADER_STRUCT + self.COMMANDS['delete']['struct'] % len(key), self.MAGIC['request'], self.COMMANDS['delete']['command'], len(key), 0, 0, 0, len(key), 0, 0, str_to_bytes(key)) msg += struct.pack( self.HEADER_STRUCT + self.COMMANDS['noop']['struct'], self.MAGIC['request'], self.COMMANDS['noop']['command'], 0, 0, 0, 0, 0, 0, 0) self._send(msg) opcode = -1 retval = True while opcode != self.COMMANDS['noop']['command']: (magic, opcode, keylen, extlen, datatype, status, bodylen, opaque, cas, extra_content) = self._get_response() if status != self.STATUS['success']: retval = False if status == self.STATUS['server_disconnected']: break return retval
def get(self, key): """ Get a key and its CAS value from server. If the value isn't cached, return (None, None). :param key: Key's name :type key: six.string_types :return: Returns (value, cas). :rtype: object """ logger.debug('Getting key %s', key) keybytes = str_to_bytes(key) data = struct.pack( self.HEADER_STRUCT + self.COMMANDS['get']['struct'] % (len(keybytes), ), self.MAGIC['request'], self.COMMANDS['get']['command'], len(keybytes), 0, 0, 0, len(keybytes), 0, 0, keybytes) self._send(data) (magic, opcode, keylen, extlen, datatype, status, bodylen, opaque, cas, extra_content) = self._get_response() logger.debug('Value Length: %d. Body length: %d. Data type: %d', extlen, bodylen, datatype) if status != self.STATUS['success']: if status == self.STATUS['key_not_found']: logger.debug('Key not found. Message: %s', extra_content) return None, None if status == self.STATUS['server_disconnected']: return None, None raise MemcachedException( 'Code: %d Message: %s' % (status, extra_content), status) flags, value = struct.unpack('!L%ds' % (bodylen - 4, ), extra_content) return self.deserialize(value, flags), cas
def get(self, key): """ Get a key and its CAS value from server. If the value isn't cached, return (None, None). :param key: Key's name :type key: six.string_type :return: Returns (value, cas). :rtype: object """ logger.info('Getting key %s' % key) data = struct.pack(self.HEADER_STRUCT + self.COMMANDS['get']['struct'] % (len(key)), self.MAGIC['request'], self.COMMANDS['get']['command'], len(key), 0, 0, 0, len(key), 0, 0, str_to_bytes(key)) self._send(data) (magic, opcode, keylen, extlen, datatype, status, bodylen, opaque, cas, extra_content) = self._get_response() logger.debug('Value Length: %d. Body length: %d. Data type: %d' % ( extlen, bodylen, datatype)) if status != self.STATUS['success']: if status == self.STATUS['key_not_found']: logger.debug('Key not found. Message: %s' % extra_content) return None, None if status == self.STATUS['server_disconnected']: return None, None raise MemcachedException('Code: %d Message: %s' % (status, extra_content)) flags, value = struct.unpack('!L%ds' % (bodylen - 4, ), extra_content) return self.deserialize(value, flags), cas
def set_multi(self, mappings, time=100, compress_level=-1): """ Set multiple keys with its values on server. If a key is a (key, cas) tuple, insert as if cas(key, value, cas) had been called. :param mappings: A dict with keys/values :type mappings: dict :param time: Time in seconds that your key will expire. :type time: int :param compress_level: How much to compress. 0 = no compression, 1 = fastest, 9 = slowest but best, -1 = default compression level. :type compress_level: int :return: True :rtype: bool """ mappings = mappings.items() msg = [] for key, value in mappings: if isinstance(key, tuple): key, cas = key else: cas = None if cas == 0: # Like cas(), if the cas value is 0, treat it as compare-and-set against not # existing. command = 'addq' else: command = 'setq' flags, value = self.serialize(value, compress_level=compress_level) m = struct.pack(self.HEADER_STRUCT + self.COMMANDS[command]['struct'] % (len(key), len(value)), self.MAGIC['request'], self.COMMANDS[command]['command'], len(key), 8, 0, 0, len(key) + len(value) + 8, 0, cas or 0, flags, time, str_to_bytes(key), value) msg.append(m) m = struct.pack(self.HEADER_STRUCT + self.COMMANDS['noop']['struct'], self.MAGIC['request'], self.COMMANDS['noop']['command'], 0, 0, 0, 0, 0, 0, 0) msg.append(m) if six.PY2: msg = ''.join(msg) else: msg = b''.join(msg) self._send(msg) opcode = -1 retval = True while opcode != self.COMMANDS['noop']['command']: (magic, opcode, keylen, extlen, datatype, status, bodylen, opaque, cas, extra_content) = self._get_response() if status != self.STATUS['success']: retval = False if status == self.STATUS['server_disconnected']: break return retval
def get_multi(self, keys): """ Get multiple keys from server. Since keys are converted to b'' when six.PY3 the keys need to be decoded back into string . e.g key='test' is read as b'test' and then decoded back to 'test' This encode/decode does not work when key is already a six.binary_type hence this function remembers which keys were originally sent as str so that it only decoded those keys back to string which were sent as string :param keys: A list of keys to from server. :type keys: list :return: A dict with all requested keys. :rtype: dict """ # pipeline N-1 getkq requests, followed by a regular getk to uncork the # server o_keys = keys keys, last = keys[:-1], str_to_bytes(keys[-1]) if six.PY2: msg = '' else: msg = b'' msg = msg.join([ struct.pack(self.HEADER_STRUCT + self.COMMANDS['getkq']['struct'] % (len(key)), self.MAGIC['request'], self.COMMANDS['getkq']['command'], len(key), 0, 0, 0, len(key), 0, 0, str_to_bytes(key)) for key in keys]) msg += struct.pack(self.HEADER_STRUCT + self.COMMANDS['getk']['struct'] % (len(last)), self.MAGIC['request'], self.COMMANDS['getk']['command'], len(last), 0, 0, 0, len(last), 0, 0, last) self._send(msg) d = {} opcode = -1 while opcode != self.COMMANDS['getk']['command']: (magic, opcode, keylen, extlen, datatype, status, bodylen, opaque, cas, extra_content) = self._get_response() if status == self.STATUS['success']: flags, key, value = struct.unpack('!L%ds%ds' % (keylen, bodylen - keylen - 4), extra_content) if six.PY2: d[key] = self.deserialize(value, flags), cas else: try: decoded_key = key.decode() except UnicodeDecodeError: d[key] = self.deserialize(value, flags), cas else: if decoded_key in o_keys: d[decoded_key] = self.deserialize(value, flags), cas else: d[key] = self.deserialize(value, flags), cas elif status == self.STATUS['server_disconnected']: break elif status != self.STATUS['key_not_found']: raise MemcachedException('Code: %d Message: %s' % (status, extra_content), status) return d
def set_multi(self, mappings, time=100, compress_level=-1): """ Set multiple keys with its values on server. If a key is a (key, cas) tuple, insert as if cas(key, value, cas) had been called. :param mappings: A dict with keys/values :type mappings: dict :param time: Time in seconds that your key will expire. :type time: int :param compress_level: How much to compress. 0 = no compression, 1 = fastest, 9 = slowest but best, -1 = default compression level. :type compress_level: int :return: List of keys that failed to be set. :rtype: list """ mappings = list(mappings.items()) msg = [] for opaque, (key, value) in enumerate(mappings): if isinstance(key, tuple): key, cas = key else: cas = None if cas == 0: # Like cas(), if the cas value is 0, treat it as compare-and-set against not # existing. command = 'addq' else: command = 'setq' keybytes = str_to_bytes(key) flags, value = self.serialize(value, compress_level=compress_level) m = struct.pack( self.HEADER_STRUCT + self.COMMANDS[command]['struct'] % (len(keybytes), len(value)), self.MAGIC['request'], self.COMMANDS[command]['command'], len(keybytes), 8, 0, 0, len(keybytes) + len(value) + 8, opaque, cas or 0, flags, time, keybytes, value) msg.append(m) m = struct.pack(self.HEADER_STRUCT + self.COMMANDS['noop']['struct'], self.MAGIC['request'], self.COMMANDS['noop']['command'], 0, 0, 0, 0, 0, 0, 0) msg.append(m) msg = b''.join(msg) self._send(msg) opcode = -1 failed = [] while opcode != self.COMMANDS['noop']['command']: (magic, opcode, keylen, extlen, datatype, status, bodylen, opaque, cas, extra_content) = self._get_response() if status == self.STATUS['server_disconnected']: # Assume that the entire operation failed. return list(key for key, value in mappings) if status != self.STATUS['success']: key, value = mappings[opaque] if isinstance(key, tuple): failed.append((key[0], cas)) else: failed.append(key) return failed