def testDeferredQueue(self):
d = DeferredQueue()
def f(i):
pass
for i in range(10):
d.defer(f, i)
Tasklet.sleep(1)
for i in range(10):
d.defer(f, i)
Tasklet.sleep(1)
class MemcacheConnection(object):
log = logging.getLogger("MemcacheConnection")
_tasklet_pool = TaskletPool(worker_timeout=2.0,
worker_timeout_relative=False)
def __init__(self, address, protocol="text", codec="default"):
self._address = address
self._stream = None
self._read_queue = DeferredQueue(self._tasklet_pool.defer)
self._write_queue = DeferredQueue(self._tasklet_pool.defer)
self._protocol = MemcacheProtocol.create(protocol)
self._protocol.set_codec(MemcacheCodec.create(codec))
def connect(self):
self._stream = BufferedStream(Socket.connect(self._address))
def is_connected(self):
return self._stream is not None
def _defer_commands(self, cmds, result_channel):
def _read_results():
protocol = self._protocol
with self._stream.get_reader() as reader:
for cmd, args, error_value in cmds:
try:
result = protocol.read(cmd, reader)
result_channel.send(result)
except TaskletExit:
raise
except:
self.log.exception("read error in defer_commands")
result_channel.send(
(MemcacheResult.ERROR, error_value))
#end _read_commands
def _write_commands():
protocol = self._protocol
try:
if not self.is_connected():
self.connect()
except TaskletExit:
raise
except:
self.log.exception("connect error in defer_commands")
for _, _, error_value in cmds:
result_channel.send((MemcacheResult.ERROR, error_value))
return
with self._stream.get_writer() as writer:
for cmd, args, error_value in cmds:
try:
protocol.write(cmd, writer, args)
except TaskletExit:
raise
except:
self.log.exception("write error in defer_commands")
result_channel.send(
(MemcacheResult.ERROR, error_value))
writer.flush()
self._read_queue.defer(_read_results)
#end _write_commands
self._write_queue.defer(_write_commands)
def _defer_command(self, cmd, args, result_channel, error_value=None):
self._defer_commands([(cmd, args, error_value)], result_channel)
def _do_command(self, cmd, args, error_value=None):
result_channel = ResultChannel()
self._defer_command(cmd, args, result_channel, error_value)
try:
return result_channel.receive()
except TimeoutError:
return MemcacheResult.TIMEOUT, error_value
def close(self):
if self.is_connected():
self._stream.close()
self._stream = None
def __setitem__(self, key, data):
self.set(key, data)
def __getitem__(self, key):
return self.get(key)
def delete(self, key, expiration=0):
return self._do_command("delete", (key, expiration))[0]
def set(self, key, data, expiration=0, flags=0):
return self._do_command("set", (key, data, expiration, flags))[0]
def add(self, key, data, expiration=0, flags=0):
return self._do_command("add", (key, data, expiration, flags))[0]
def replace(self, key, data, expiration=0, flags=0):
return self._do_command("replace", (key, data, expiration, flags))[0]
def append(self, key, data, expiration=0, flags=0):
return self._do_command("append", (key, data, expiration, flags))[0]
def prepend(self, key, data, expiration=0, flags=0):
return self._do_command("prepend", (key, data, expiration, flags))[0]
def cas(self, key, data, cas_unique, expiration=0, flags=0):
return self._do_command("cas",
(key, data, expiration, flags, cas_unique))[0]
def incr(self, key, increment):
return self._do_command("incr", (key, increment))
def decr(self, key, increment):
return self._do_command("decr", (key, increment))
def get(self, key, default=None):
_, values = self._do_command("get", ([key], ), {})
return values.get(key, default)
def getr(self, key, default=None):
result, values = self._do_command("get", ([key], ), {})
return result, values.get(key, default)
def gets(self, key, default=None):
result, values = self._do_command("gets", ([key], ), {})
value, cas_unique = values.get(key, (default, None))
return result, value, cas_unique
def get_multi(self, keys):
return self._do_command("get", (keys, ))
def gets_multi(self, keys):
return self._do_command("gets", (keys, ))
def version(self):
return self._do_command("version", ())
def batch(self):
return CommandBatch(self)
class MemcacheConnection(object):
log = logging.getLogger("MemcacheConnection")
_tasklet_pool = TaskletPool(worker_timeout = 2.0,
worker_timeout_relative = False)
def __init__(self, address, protocol = "text", codec = "default"):
self._address = address
self._stream = None
self._read_queue = DeferredQueue(self._tasklet_pool.defer)
self._write_queue = DeferredQueue(self._tasklet_pool.defer)
self._protocol = MemcacheProtocol.create(protocol)
self._protocol.set_codec(MemcacheCodec.create(codec))
def connect(self):
self._stream = BufferedStream(Socket.connect(self._address))
def is_connected(self):
return self._stream is not None
def _defer_commands(self, cmds, result_channel):
def _read_results():
protocol = self._protocol
with self._stream.get_reader() as reader:
for cmd, args, error_value in cmds:
try:
result = protocol.read(cmd, reader)
result_channel.send(result)
except TaskletExit:
raise
except:
self.log.exception("read error in defer_commands")
result_channel.send((MemcacheResult.ERROR, error_value))
#end _read_commands
def _write_commands():
protocol = self._protocol
try:
if not self.is_connected():
self.connect()
except TaskletExit:
raise
except:
self.log.exception("connect error in defer_commands")
for _, _, error_value in cmds:
result_channel.send((MemcacheResult.ERROR, error_value))
return
with self._stream.get_writer() as writer:
for cmd, args, error_value in cmds:
try:
protocol.write(cmd, writer, args)
except TaskletExit:
raise
except:
self.log.exception("write error in defer_commands")
result_channel.send((MemcacheResult.ERROR, error_value))
writer.flush()
self._read_queue.defer(_read_results)
#end _write_commands
self._write_queue.defer(_write_commands)
def _defer_command(self, cmd, args, result_channel, error_value = None):
self._defer_commands([(cmd, args, error_value)], result_channel)
def _do_command(self, cmd, args, error_value = None):
result_channel = ResultChannel()
self._defer_command(cmd, args, result_channel, error_value)
try:
return result_channel.receive()
except TimeoutError:
return MemcacheResult.TIMEOUT, error_value
def close(self):
if self.is_connected():
self._stream.close()
self._stream = None
def __setitem__(self, key, data):
self.set(key, data)
def __getitem__(self, key):
return self.get(key)
def delete(self, key, expiration = 0):
return self._do_command("delete", (key, expiration))[0]
def set(self, key, data, expiration = 0, flags = 0):
return self._do_command("set", (key, data, expiration, flags))[0]
def add(self, key, data, expiration = 0, flags = 0):
return self._do_command("add", (key, data, expiration, flags))[0]
def replace(self, key, data, expiration = 0, flags = 0):
return self._do_command("replace", (key, data, expiration, flags))[0]
def append(self, key, data, expiration = 0, flags = 0):
return self._do_command("append", (key, data, expiration, flags))[0]
def prepend(self, key, data, expiration = 0, flags = 0):
return self._do_command("prepend", (key, data, expiration, flags))[0]
def cas(self, key, data, cas_unique, expiration = 0, flags = 0):
return self._do_command("cas", (key, data, expiration, flags, cas_unique))[0]
def incr(self, key, increment):
return self._do_command("incr", (key, increment))
def decr(self, key, increment):
return self._do_command("decr", (key, increment))
def get(self, key, default = None):
_, values = self._do_command("get", ([key], ), {})
return values.get(key, default)
def getr(self, key, default = None):
result, values = self._do_command("get", ([key], ), {})
return result, values.get(key, default)
def gets(self, key, default = None):
result, values = self._do_command("gets", ([key], ), {})
value, cas_unique = values.get(key, (default, None))
return result, value, cas_unique
def get_multi(self, keys):
return self._do_command("get", (keys, ))
def gets_multi(self, keys):
return self._do_command("gets", (keys, ))
def version(self):
return self._do_command("version", ())
def batch(self):
return CommandBatch(self)