def handle(self, socket, application):
stream = BufferedStream(socket)
#implements http1.1 keep alive handler
#there are several concurrent tasks for each connection;
#1 for reading requests, 1 or more for handling requests and 1 for writing responses
#the current task (the one created to handle the socket connection) is the controller task,
#e.g. it coordinates the actions of it's children by message passing
control = Tasklet.current()
#writes responses back to the client when they are ready:
response_writer = Tasklet.new(self.write_responses,
name='response_writer')(control, stream)
#reads requests from clients:
request_reader = Tasklet.new(self.read_requests,
name='request_reader')(control, stream)
#typical flow:
#1. reader reads in request, sends notification to control (MSG_REQUEST_READ)
#2. control starts handler for the request
#3. handler works on request and sends notification to control when finished (MSG_REQUEST_HANDLED)
#4. control sends message to writer to start writing the response (MSG_WRITE_RESPONSE)
#5. writer notififies control when response is wriiten (MSG_RESPONSE_WRITTEN)
#control wait for msgs to arrive:
for msg, (request, response), kwargs in Tasklet.receive():
if msg.match(self.MSG_REQUEST_READ):
#we use reque to be able to send the responses back in the correct order later
self._reque.start(request)
Tasklet.new(self.handle_request,
name='request_handler')(control, request,
application)
elif msg.match(self.MSG_REQUEST_HANDLED):
#we use reque to retire (send out) the responses in the correct order
for request, response in self._reque.finish(request, response):
self.MSG_WRITE_RESPONSE.send(response_writer)(request,
response)
elif msg.match(self.MSG_RESPONSE_WRITTEN):
if request.version == 'HTTP/1.0':
break #no keep-alive support in http 1.0
elif request.get_response_header('Connection') == 'close':
break #response indicated to close after response
elif request.get_request_header('Connection') == 'close':
break #request indicated to close after response
elif msg.match(self.MSG_READ_ERROR):
break #stop and close the connection
elif msg.match(self.MSG_WRITE_ERROR):
break #stop and close the connection
else:
assert False, "unexpected msg in control loop"
#kill reader and writer
#any outstanding request will continue, but will exit by themselves
response_writer.kill()
request_reader.kill()
#close our side of the socket
stream.close()
class RemoteClient(object):
"""Remoteing client. This represents the connection to the remote server.
Use the lookup method to get a reference to a remote task.
This reference can then be used to send or call msgs to the remote tasks
"""
def __init__(self):
self._stream = None
self._message_writer_task = None
self._message_reader_task = None
self._bootstrap_task = RemoteTasklet(self, 0) # proxy to the remote bootstrap service
self._blocked_message = {} # message_id -> message, for keeping track of blocking calls
def _message_writer(self):
object_writer = ObjectWriter(self._stream.writer)
for msg, (remote_task_id, args), kwargs in Tasklet.receive():
object_writer.write_object((remote_task_id, msg.__class__, id(msg), args, kwargs))
object_writer.flush()
def _message_reader(self):
object_reader = ObjectReader(self._stream.reader)
while True:
msg, args = object_reader.read_object()
if issubclass(msg, MSG_RESULT):
msg_id, result = args
if msg_id in self._blocked_message:
self._blocked_message[msg_id].reply(result)
else:
pass # this happens when caller already gone due to timeout
def connect(self, endpoint):
self._stream = BufferedStream(Connector.connect(endpoint))
self._message_writer_task = Tasklet.new(self._message_writer)()
self._message_reader_task = Tasklet.new(self._message_reader)()
def close(self):
self._message_writer_task.kill()
self._message_reader_task.kill()
self._stream.close()
def send(self, remote_task_id, msg, args, kwargs):
self._message_writer_task.send(msg, remote_task_id, args)
def call(self, remote_task_id, timeout, msg, args, kwargs):
msg_id = id(msg)
self._blocked_message[msg_id] = msg
try:
self.send(remote_task_id, msg, args, kwargs)
result = msg.wait(timeout)
return result
finally:
del self._blocked_message[msg_id]
def lookup(self, name):
remote_task_id = MSG_LOOKUP.call(self._bootstrap_task)(name)
if remote_task_id > 0:
return RemoteTasklet(self, remote_task_id)
else:
return None
def handle(client_socket):
stream = BufferedStream(client_socket)
with stream.get_writer() as writer:
writer.write_bytes("HTTP/1.0 200 OK\r\n")
writer.write_bytes("Content-Length: 12\r\n")
writer.write_bytes("\r\n")
writer.write_bytes("Hello World!")
writer.flush()
stream.close()
def handle(client_socket):
stream = BufferedStream(client_socket)
with stream.get_writer() as writer:
writer.write_bytes("HTTP/1.0 200 OK\r\n")
writer.write_bytes("Content-Length: 12\r\n")
writer.write_bytes("\r\n")
writer.write_bytes("Hello World!")
writer.flush()
stream.close()
def connect(self, endpoint):
"""Connect to the webserver at *endpoint*. *endpoint* is a tuple (<host>, <port>)."""
self._host = None
if type(endpoint) == type(()):
try:
self._host = endpoint[0]
except:
pass
self._stream = BufferedStream(Connector.connect(endpoint))
def handler(client_socket):
"""writes the familiar greeting to client"""
stream = BufferedStream(client_socket)
writer = stream.writer
writer.write_bytes("HTTP/1.0 200 OK\r\n")
writer.write_bytes("Content-Length: 12\r\n")
writer.write_bytes("\r\n")
writer.write_bytes("Hello World!")
writer.flush()
stream.close()
def __init__(self, devname):
self.fd = fd = os.open(devname, os.O_RDWR | os.O_NOCTTY | os.O_NONBLOCK | os.O_EXCL)
attr = termios.tcgetattr(fd)
attr[0] = attr[0] & ~(termios.IGNBRK | termios.BRKINT | termios.PARMRK | termios.ISTRIP | termios.INLCR | termios.IGNCR | termios.ICRNL | termios.IXON);
attr[1] = attr[1] & ~termios.OPOST;
attr[2] = attr[2] & ~(termios.CSIZE | termios.PARENB) | termios.CS8;
attr[3] = attr[3] & ~(termios.ECHO | termios.ECHONL | termios.ICANON | termios.ISIG | termios.IEXTEN);
attr[4] = termios.B19200
termios.tcsetattr(fd, termios.TCSANOW, attr)
BufferedStream.__init__(self, FDStream(fd))
def handler(client_socket):
"""writes the familiar greeting to client"""
stream = BufferedStream(client_socket)
writer = stream.writer
writer.write_bytes("HTTP/1.0 200 OK\r\n")
writer.write_bytes("Content-Length: 12\r\n")
writer.write_bytes("\r\n")
writer.write_bytes("Hello World!")
writer.flush()
stream.close()
def handle(self, socket, application):
stream = BufferedStream(socket)
#implements http1.1 keep alive handler
#there are several concurrent tasks for each connection;
#1 for reading requests, 1 or more for handling requests and 1 for writing responses
#the current task (the one created to handle the socket connection) is the controller task,
#e.g. it coordinates the actions of it's children by message passing
control = Tasklet.current()
#writes responses back to the client when they are ready:
response_writer = Tasklet.new(self.write_responses, name = 'response_writer')(control, stream)
#reads requests from clients:
request_reader = Tasklet.new(self.read_requests, name = 'request_reader')(control, stream)
#typical flow:
#1. reader reads in request, sends notification to control (MSG_REQUEST_READ)
#2. control starts handler for the request
#3. handler works on request and sends notification to control when finished (MSG_REQUEST_HANDLED)
#4. control sends message to writer to start writing the response (MSG_WRITE_RESPONSE)
#5. writer notififies control when response is wriiten (MSG_RESPONSE_WRITTEN)
#control wait for msgs to arrive:
for msg, (request, response), kwargs in Tasklet.receive():
if msg.match(self.MSG_REQUEST_READ):
#we use reque to be able to send the responses back in the correct order later
self._reque.start(request)
Tasklet.new(self.handle_request, name = 'request_handler')(control, request, application)
elif msg.match(self.MSG_REQUEST_HANDLED):
#request.environ["wsgi.input"].read(request.environ["wsgi.input"]._n)
#we use reque to retire (send out) the responses in the correct order
for request, response in self._reque.finish(request, response):
self.MSG_WRITE_RESPONSE.send(response_writer)(request, response)
elif msg.match(self.MSG_RESPONSE_WRITTEN):
if request.version == 'HTTP/1.0':
break #no keep-alive support in http 1.0
elif request.get_response_header('Connection') == 'close':
break #response indicated to close after response
elif request.get_request_header('Connection') == 'close':
break #request indicated to close after response
elif msg.match(self.MSG_READ_ERROR):
break #stop and close the connection
elif msg.match(self.MSG_WRITE_ERROR):
break #stop and close the connection
else:
assert False, "unexpected msg in control loop"
#kill reader and writer
#any outstanding request will continue, but will exit by themselves
response_writer.kill()
request_reader.kill()
#close our side of the socket
stream.close()
def connect(self, endpoint):
"""Connect to the webserver at *endpoint*. *endpoint* is a tuple (<host>, <port>)."""
self._host = None
if type(endpoint) == type(()):
try:
self._host = endpoint[0]
except Exception:
pass
self._stream = BufferedStream(Connector.connect(endpoint),
read_buffer_size=1024 * 8,
write_buffer_size=1024 * 4)
def handler(client_socket):
"""writes the familiar greeting to client"""
stream = BufferedStream(client_socket)
writer = stream.writer
writer.write_bytes("ok\r\n")
reader = stream.reader
buf = reader.read_bytes()
print buf
writer.write_bytes(buf + ':' + compute(buf) + '\n')
writer.flush()
stream.close()
def handler(client_socket):
"""writes the familiar greeting to client"""
stream = BufferedStream(client_socket)
writer = stream.writer
writer.write_bytes("ok\r\n")
reader = stream.reader
buf = reader.read_bytes()
print buf
writer.write_bytes(buf + ':' + compute(buf) + '\n')
writer.flush()
stream.close()
def handler(client_socket):
"""writes the familiar greeting to client"""
cur_thread = threading.current_thread()
stream = BufferedStream(client_socket)
writer = stream.writer
writer.write_bytes("HTTP/1.0 200 OK\r\n")
writer.write_bytes("Content-Length: 12\r\n")
writer.write_bytes("\r\n")
writer.write_bytes("[%s:%s] - " % (cur_thread, Tasklet.current()))
writer.flush()
stream.close()
def start(self, client_socket):
''' Start the task: start one tasklet listing for incomming connections. '''
#self.__receiving_tasklet = Tasklet.new(self.__serve)()
self.__client_socket = client_socket
self.__stream = BufferedStream(self.__client_socket)
q.logger.log("[SocketTaskHandler] Client connected.", 3)
self.__sending_tasklet = Tasklet.new(self.__send)()
self.__receiving_tasklet = Tasklet.new(self.__receive())()
def truncator():
while len(clients) < N:
Tasklet.sleep(1.0)
Tasklet.sleep(5.0)
while True:
print 'connecting truncator'
client = Socket.connect(('localhost', 6379))
stream = BufferedStream(client)
print 'issueing trunc of append only log'
bgrewriteaof(stream)
stream.close()
Tasklet.sleep(120.0)
def truncator():
while len(clients) < N:
Tasklet.sleep(1.0)
Tasklet.sleep(5.0)
while True:
print 'connecting truncator'
client = Socket.connect(('localhost', 6379))
stream = BufferedStream(client)
print 'issueing trunc of append only log'
bgrewriteaof(stream)
stream.close()
Tasklet.sleep(120.0)
def __init__(self, devname):
self.fd = fd = os.open(
devname, os.O_RDWR | os.O_NOCTTY | os.O_NONBLOCK | os.O_EXCL)
attr = termios.tcgetattr(fd)
attr[0] = attr[0] & ~(termios.IGNPAR | termios.IGNBRK | termios.BRKINT
| termios.PARMRK | termios.ISTRIP | termios.INLCR
| termios.IGNCR | termios.ICRNL | termios.IXON)
attr[1] = attr[1] & ~termios.OPOST
attr[2] = attr[2] & ~(termios.CSIZE | termios.PARENB) | termios.CS8
attr[3] = attr[3] & ~(termios.ECHO | termios.ECHONL | termios.ICANON
| termios.ISIG | termios.IEXTEN)
attr[4] = attr[5] = termios.B19200
termios.tcsetattr(fd, termios.TCSANOW, attr)
BufferedStream.__init__(self, FDStream(fd))
def connect(self, endpoint):
"""Connect to the webserver at *endpoint*. *endpoint* is a tuple (<host>, <port>)."""
self._host = None
if type(endpoint) == type(()):
try:
self._host = endpoint[0]
except:
pass
self._stream = BufferedStream(Connector.connect(endpoint))
def connect(self, endpoint):
"""Connect to the webserver at *endpoint*. *endpoint* is a tuple (<host>, <port>)."""
self._host = None
if type(endpoint) == type(()):
try:
self._host = endpoint[0]
except Exception:
pass
self._stream = BufferedStream(Connector.connect(endpoint), read_buffer_size = 1024 * 8, write_buffer_size = 1024 * 4)
def handler(client_socket):
print >> sys.stderr, 'info: connection from %r' % (
client_socket.socket.getpeername(), )
stream = BufferedStream(client_socket)
reader = stream.reader # Strips \r\n and \n from the end.
writer = stream.writer
# Read HTTP request.
line1 = None
try:
while True:
line = reader.read_line()
if not line: # Empty line, end of HTTP request.
break
if line1 is None:
line1 = line
except EOFError:
pass
# Parse HTTP request.
# Please note that an assertion here doesn't abort the server.
items = line1.split(' ')
assert 3 == len(items)
assert items[2] in ('HTTP/1.0', 'HTTP/1.1')
assert items[0] == 'GET'
assert items[1].startswith('/')
try:
num = int(items[1][1:])
except ValueError:
num = None
# Write HTTP response.
if num is None:
writer.write_bytes(
'HTTP/1.0 200 OK\r\nContent-Type: text/html\r\n\r\n')
writer.write_bytes('<a href="/0">start at 0</a><p>Hello, World!\n')
else:
next_num = lprng.Lprng(num).next()
writer.write_bytes(
'HTTP/1.0 200 OK\r\nContent-Type: text/html\r\n\r\n')
writer.write_bytes('<a href="/%d">continue with %d</a>\n' %
(next_num, next_num))
writer.flush()
stream.close()
def start(self, client_socket):
''' Start the task: start one tasklet listing for incomming connections. '''
#self.__receiving_tasklet = Tasklet.new(self.__serve)()
self.__client_socket = client_socket
self.__stream = BufferedStream(self.__client_socket)
q.logger.log("[SocketTaskHandler] Client connected.", 3)
self.__sending_tasklet = Tasklet.new(self.__send)()
self.__receiving_tasklet = Tasklet.new(self.__receive())()
def handler(client_socket):
print >>sys.stderr, "info: connection from %r" % (client_socket.socket.getpeername(),)
stream = BufferedStream(client_socket)
reader = stream.reader # Strips \r\n and \n from the end.
writer = stream.writer
# Read HTTP request.
line1 = None
try:
while True:
line = reader.read_line()
if not line: # Empty line, end of HTTP request.
break
if line1 is None:
line1 = line
except EOFError:
pass
# Parse HTTP request.
# Please note that an assertion here doesn't abort the server.
items = line1.split(" ")
assert 3 == len(items)
assert items[2] in ("HTTP/1.0", "HTTP/1.1")
assert items[0] == "GET"
assert items[1].startswith("/")
try:
num = int(items[1][1:])
except ValueError:
num = None
# Write HTTP response.
if num is None:
writer.write_bytes("HTTP/1.0 200 OK\r\nContent-Type: text/html\r\n\r\n")
writer.write_bytes('<a href="/0">start at 0</a><p>Hello, World!\n')
else:
next_num = lprng.Lprng(num).next()
writer.write_bytes("HTTP/1.0 200 OK\r\nContent-Type: text/html\r\n\r\n")
writer.write_bytes('<a href="/%d">continue with %d</a>\n' % (next_num, next_num))
writer.flush()
stream.close()
def handle(client_socket):
"""handles a single client connected to the chat server"""
stream = BufferedStream(client_socket)
client_task = Tasklet.current() #this is the current task as started by server
connected_clients.add(client_task)
def writer():
for msg, args, kwargs in Tasklet.receive():
if msg.match(MSG_WRITE_LINE):
stream.writer.write_bytes(args[0] + '\n')
stream.writer.flush()
def reader():
for line in stream.reader.read_lines():
line = line.strip()
if line == 'quit':
MSG_QUIT.send(client_task)()
else:
MSG_LINE_READ.send(client_task)(line)
reader_task = Tasklet.new(reader)()
writer_task = Tasklet.new(writer)()
MSG_WRITE_LINE.send(writer_task)("type 'quit' to exit..")
for msg, args, kwargs in Tasklet.receive():
if msg.match(MSG_QUIT):
break
elif msg.match(MSG_LINE_READ):
#a line was recv from our client, multicast it to the other clients
for task in connected_clients:
if task != client_task: #don't echo the line back to myself
MSG_WRITE_LINE.send(task)(args[0])
elif msg.match(MSG_WRITE_LINE):
MSG_WRITE_LINE.send(writer_task)(args[0])
connected_clients.remove(client_task)
reader_task.kill()
writer_task.kill()
client_socket.close()
class MemcacheConnection(object):
log = logging.getLogger("MemcacheConnection")
_tasklet_pool = TaskletPool(worker_timeout = 2.0,
worker_timeout_relative = False)
def __init__(self, address, protocol = "text", codec = "default"):
self._address = address
self._stream = None
self._read_queue = DeferredQueue(self._tasklet_pool.defer)
self._write_queue = DeferredQueue(self._tasklet_pool.defer)
self._protocol = MemcacheProtocol.create(protocol)
self._protocol.set_codec(MemcacheCodec.create(codec))
def connect(self):
self._stream = BufferedStream(Socket.connect(self._address))
def is_connected(self):
return self._stream is not None
def _defer_commands(self, cmds, result_channel):
def _read_results():
protocol = self._protocol
with self._stream.get_reader() as reader:
for cmd, args, error_value in cmds:
try:
result = protocol.read(cmd, reader)
result_channel.send(result)
except TaskletExit:
raise
except:
self.log.exception("read error in defer_commands")
result_channel.send((MemcacheResult.ERROR, error_value))
#end _read_commands
def _write_commands():
protocol = self._protocol
try:
if not self.is_connected():
self.connect()
except TaskletExit:
raise
except:
self.log.exception("connect error in defer_commands")
for _, _, error_value in cmds:
result_channel.send((MemcacheResult.ERROR, error_value))
return
with self._stream.get_writer() as writer:
for cmd, args, error_value in cmds:
try:
protocol.write(cmd, writer, args)
except TaskletExit:
raise
except:
self.log.exception("write error in defer_commands")
result_channel.send((MemcacheResult.ERROR, error_value))
writer.flush()
self._read_queue.defer(_read_results)
#end _write_commands
self._write_queue.defer(_write_commands)
def _defer_command(self, cmd, args, result_channel, error_value = None):
self._defer_commands([(cmd, args, error_value)], result_channel)
def _do_command(self, cmd, args, error_value = None):
result_channel = ResultChannel()
self._defer_command(cmd, args, result_channel, error_value)
try:
return result_channel.receive()
except TimeoutError:
return MemcacheResult.TIMEOUT, error_value
def close(self):
if self.is_connected():
self._stream.close()
self._stream = None
def __setitem__(self, key, data):
self.set(key, data)
def __getitem__(self, key):
return self.get(key)
def delete(self, key, expiration = 0):
return self._do_command("delete", (key, expiration))[0]
def set(self, key, data, expiration = 0, flags = 0):
return self._do_command("set", (key, data, expiration, flags))[0]
def add(self, key, data, expiration = 0, flags = 0):
return self._do_command("add", (key, data, expiration, flags))[0]
def replace(self, key, data, expiration = 0, flags = 0):
return self._do_command("replace", (key, data, expiration, flags))[0]
def append(self, key, data, expiration = 0, flags = 0):
return self._do_command("append", (key, data, expiration, flags))[0]
def prepend(self, key, data, expiration = 0, flags = 0):
return self._do_command("prepend", (key, data, expiration, flags))[0]
def cas(self, key, data, cas_unique, expiration = 0, flags = 0):
return self._do_command("cas", (key, data, expiration, flags, cas_unique))[0]
def incr(self, key, increment):
return self._do_command("incr", (key, increment))
def decr(self, key, increment):
return self._do_command("decr", (key, increment))
def get(self, key, default = None):
_, values = self._do_command("get", ([key], ), {})
return values.get(key, default)
def getr(self, key, default = None):
result, values = self._do_command("get", ([key], ), {})
return result, values.get(key, default)
def gets(self, key, default = None):
result, values = self._do_command("gets", ([key], ), {})
value, cas_unique = values.get(key, (default, None))
return result, value, cas_unique
def get_multi(self, keys):
return self._do_command("get", (keys, ))
def gets_multi(self, keys):
return self._do_command("gets", (keys, ))
def version(self):
return self._do_command("version", ())
def batch(self):
return CommandBatch(self)
class RemoteClient(object):
"""Remoteing client. This represents the connection to the remote server.
Use the lookup method to get a reference to a remote task.
This reference can then be used to send or call msgs to the remote tasks
"""
def __init__(self):
self._stream = None
self._message_writer_task = None
self._message_reader_task = None
self._bootstrap_task = RemoteTasklet(
self, 0) #proxy to the remote bootstrap service
self._blocked_message = {
} #message_id -> message, for keeping track of blocking calls
def _message_writer(self):
object_writer = ObjectWriter(self._stream.writer)
for msg, (remote_task_id, args), kwargs in Tasklet.receive():
object_writer.write_object(
(remote_task_id, msg.__class__, id(msg), args, kwargs))
object_writer.flush()
def _message_reader(self):
object_reader = ObjectReader(self._stream.reader)
while True:
msg, args = object_reader.read_object()
if issubclass(msg, MSG_RESULT):
msg_id, result = args
if msg_id in self._blocked_message:
self._blocked_message[msg_id].reply(result)
else:
pass #this happens when caller already gone due to timeout
def connect(self, endpoint):
self._stream = BufferedStream(Connector.connect(endpoint))
self._message_writer_task = Tasklet.new(self._message_writer)()
self._message_reader_task = Tasklet.new(self._message_reader)()
def close(self):
self._message_writer_task.kill()
self._message_reader_task.kill()
self._stream.close()
def send(self, remote_task_id, msg, args, kwargs):
self._message_writer_task.send(msg, remote_task_id, args)
def call(self, remote_task_id, timeout, msg, args, kwargs):
msg_id = id(msg)
self._blocked_message[msg_id] = msg
try:
self.send(remote_task_id, msg, args, kwargs)
result = msg.wait(timeout)
return result
finally:
del self._blocked_message[msg_id]
def lookup(self, name):
remote_task_id = MSG_LOOKUP.call(self._bootstrap_task)(name)
if remote_task_id > 0:
return RemoteTasklet(self, remote_task_id)
else:
return None
def connect(self):
self._stream = BufferedStream(Socket.connect(self._address))
class MemcacheConnection(object):
log = logging.getLogger("MemcacheConnection")
_tasklet_pool = TaskletPool(worker_timeout=2.0,
worker_timeout_relative=False)
def __init__(self, address, protocol="text", codec="default"):
self._address = address
self._stream = None
self._read_queue = DeferredQueue(self._tasklet_pool.defer)
self._write_queue = DeferredQueue(self._tasklet_pool.defer)
self._protocol = MemcacheProtocol.create(protocol)
self._protocol.set_codec(MemcacheCodec.create(codec))
def connect(self):
self._stream = BufferedStream(Socket.connect(self._address))
def is_connected(self):
return self._stream is not None
def _defer_commands(self, cmds, result_channel):
def _read_results():
protocol = self._protocol
with self._stream.get_reader() as reader:
for cmd, args, error_value in cmds:
try:
result = protocol.read(cmd, reader)
result_channel.send(result)
except TaskletExit:
raise
except:
self.log.exception("read error in defer_commands")
result_channel.send(
(MemcacheResult.ERROR, error_value))
#end _read_commands
def _write_commands():
protocol = self._protocol
try:
if not self.is_connected():
self.connect()
except TaskletExit:
raise
except:
self.log.exception("connect error in defer_commands")
for _, _, error_value in cmds:
result_channel.send((MemcacheResult.ERROR, error_value))
return
with self._stream.get_writer() as writer:
for cmd, args, error_value in cmds:
try:
protocol.write(cmd, writer, args)
except TaskletExit:
raise
except:
self.log.exception("write error in defer_commands")
result_channel.send(
(MemcacheResult.ERROR, error_value))
writer.flush()
self._read_queue.defer(_read_results)
#end _write_commands
self._write_queue.defer(_write_commands)
def _defer_command(self, cmd, args, result_channel, error_value=None):
self._defer_commands([(cmd, args, error_value)], result_channel)
def _do_command(self, cmd, args, error_value=None):
result_channel = ResultChannel()
self._defer_command(cmd, args, result_channel, error_value)
try:
return result_channel.receive()
except TimeoutError:
return MemcacheResult.TIMEOUT, error_value
def close(self):
if self.is_connected():
self._stream.close()
self._stream = None
def __setitem__(self, key, data):
self.set(key, data)
def __getitem__(self, key):
return self.get(key)
def delete(self, key, expiration=0):
return self._do_command("delete", (key, expiration))[0]
def set(self, key, data, expiration=0, flags=0):
return self._do_command("set", (key, data, expiration, flags))[0]
def add(self, key, data, expiration=0, flags=0):
return self._do_command("add", (key, data, expiration, flags))[0]
def replace(self, key, data, expiration=0, flags=0):
return self._do_command("replace", (key, data, expiration, flags))[0]
def append(self, key, data, expiration=0, flags=0):
return self._do_command("append", (key, data, expiration, flags))[0]
def prepend(self, key, data, expiration=0, flags=0):
return self._do_command("prepend", (key, data, expiration, flags))[0]
def cas(self, key, data, cas_unique, expiration=0, flags=0):
return self._do_command("cas",
(key, data, expiration, flags, cas_unique))[0]
def incr(self, key, increment):
return self._do_command("incr", (key, increment))
def decr(self, key, increment):
return self._do_command("decr", (key, increment))
def get(self, key, default=None):
_, values = self._do_command("get", ([key], ), {})
return values.get(key, default)
def getr(self, key, default=None):
result, values = self._do_command("get", ([key], ), {})
return result, values.get(key, default)
def gets(self, key, default=None):
result, values = self._do_command("gets", ([key], ), {})
value, cas_unique = values.get(key, (default, None))
return result, value, cas_unique
def get_multi(self, keys):
return self._do_command("get", (keys, ))
def gets_multi(self, keys):
return self._do_command("gets", (keys, ))
def version(self):
return self._do_command("version", ())
def batch(self):
return CommandBatch(self)
def connector():
for i in range(N):
client = Socket.connect(('localhost', 6379))
clients.append(BufferedStream(client))
print i
print 'connector done', N
class SocketTaskHandler(object):
'''
The SocketTask starts a tasklet that listens for incomming connections. Only one connection will be granted access.
When a client is connected, a second tasklet will be started. One tasklet is responsible for receiving messages, the other for sending messages.
The SocketTask can receive and send dicts: the dicts are serialized and deserialized using yaml, messages are seperated using '\n---\n'.
When the task receives a message, it will create a new tasklet and call the messageHandler, passing it the received data as a parameter.
'''
def __init__(self):
#self.__server_socket = Socket.new()
#self.__server_socket.set_reuse_address(1)
#self.__server_socket.bind(('', port))
#self.__server_socket.listen()
self.__client_socket = None
self.__receiving_tasklet = None
self.__sending_tasklet = None
def setMessageHandler(self, messageHandler):
'''
Set the callback that will be called when a message is received. The callback will be passed a dictionary.
'''
self.__messageHandler = messageHandler
def start(self, client_socket):
''' Start the task: start one tasklet listing for incomming connections. '''
#self.__receiving_tasklet = Tasklet.new(self.__serve)()
self.__client_socket = client_socket
self.__stream = BufferedStream(self.__client_socket)
q.logger.log("[SocketTaskHandler] Client connected.", 3)
self.__sending_tasklet = Tasklet.new(self.__send)()
self.__receiving_tasklet = Tasklet.new(self.__receive())()
def sendData(self, data):
'''
Send data over the socket.
@param data: the data to send
@raise Exception: if no client is connected
'''
if self.__sending_tasklet == None:
q.logger.log(
"[SocketTaskHandler] No connection to client, can't send the message.",
1)
raise Exception("No connection to client, can't send the message.")
else:
MSG_SOCKET_SEND.send(self.__sending_tasklet)(data)
def stop(self):
self.__stream and self.__stream.close()
def __receive(self):
# Called in the receiving tasklet
reader = self.__stream.reader
try:
buffer = ""
processData = True
while True:
line = reader.read_line()
if line <> '---':
buffer += line + "\n"
else:
try:
data = yaml.load(buffer)
processData = True
if data == 'ping':
processData = False
self.sendData('pong')
except yaml.parser.ParserError:
q.logger.log(
"[SocketTaskHandler] Received bad formatted data: "
+ str(buffer), 3)
else:
if processData:
q.logger.log(
"[SocketTaskHandler] Received data: " +
str(data), 5)
Tasklet.new(self.__messageHandler)(data, self)
buffer = ""
except EOFError:
q.logger.log("[SocketTaskHandler] Client disconnected.", 3)
MSG_SOCKET_CLOSE.send(self.__sending_tasklet)()
self.__stream.close()
def __send(self):
# Started in the sending tasklet
writer = self.__stream.writer
for msg, args, kwargs in Tasklet.receive():
if msg.match(MSG_SOCKET_SEND):
message = self.__yaml_message(args[0])
q.logger.log("[SocketTaskHandler] Sending message: " + message,
5)
writer.write_bytes(message)
writer.flush()
elif msg.match(MSG_SOCKET_CLOSE):
return
def __yaml_message(self, dict):
return yaml.dump(dict) + "\n---\n"
def connect(self, endpoint):
self._stream = BufferedStream(Connector.connect(endpoint))
self._message_writer_task = Tasklet.new(self._message_writer)()
self._message_reader_task = Tasklet.new(self._message_reader)()
class HTTPConnection(object):
"""A HTTP 1.1 Client.
Usage::
#create an instance of this class and connect to a webserver using the connect method:
cnn = HTTPConnection()
cnn.connect(('www.google.com', 80))
#create a GET request using the get method:
request = cnn.get('/index.html')
#finally perform the request to get a response:
response = cnn.perform(request)
#do something with the response:
print response.body
"""
log = logging.getLogger('HTTPConnection')
def connect(self, endpoint):
"""Connect to the webserver at *endpoint*. *endpoint* is a tuple (<host>, <port>)."""
self._host = None
if type(endpoint) == type(()):
try:
self._host = endpoint[0]
except:
pass
self._stream = BufferedStream(Connector.connect(endpoint))
def receive(self):
"""Receive the next :class:`HTTPResponse` from the connection."""
try:
return self._receive()
except TaskletExit:
raise
except EOFError:
raise HTTPError("EOF while reading response")
except Exception:
self.log.exception('')
raise HTTPError("Exception while reading response")
def _receive(self):
response = HTTPResponse()
reader = self._stream.reader
lines = reader.read_lines()
#parse status line
response.status = lines.next()
#rest of response headers
for line in lines:
if not line: break
key, value = line.split(': ')
response.add_header(key, value)
#read data
transfer_encoding = response.get_header('Transfer-Encoding', None)
try:
content_length = int(response.get_header('Content-Length'))
except:
content_length = None
#TODO better support large data
chunks = []
if transfer_encoding == 'chunked':
while True:
chunk_line = reader.read_line()
chunk_size = int(chunk_line.split(';')[0], 16)
if chunk_size > 0:
data = reader.read_bytes(chunk_size)
reader.read_line(
) #chunk is always followed by a single empty line
chunks.append(data)
else:
reader.read_line(
) #chunk is always followed by a single empty line
break
elif content_length is not None:
while content_length > 0:
n = min(CHUNK_SIZE, content_length)
data = reader.read_bytes(n)
chunks.append(data)
content_length -= len(data)
else:
assert False, 'TODO'
response.iter = chunks
return response
def get(self, path, host=None):
"""Returns a new :class:`HTTPRequest` with request.method = 'GET' and request.path = *path*.
request.host will be set to the host used in :func:`connect`, or optionally you can specify a
specific *host* just for this request.
"""
request = HTTPRequest()
request.method = 'GET'
request.path = path
request.host = host or self._host
return request
def post(self, path, body=None, host=None):
"""Returns a new :class:`HTTPRequest` with request.method = 'POST' and request.path = *path*.
request.host will be set to the host used in :func:`connect`, or optionally you can specify a
specific *host* just for this request.
*body* is an optional string containing the data to post to the server.
"""
request = HTTPRequest()
request.method = 'POST'
request.path = path
request.host = host or self._host
if body is not None:
request.body = body
return request
def perform(self, request):
"""Sends the *request* and waits for and returns the :class:`HTTPResult`."""
self.send(request)
return self.receive()
def send(self, request):
"""Sends the *request* on this connection."""
if request.method is None:
assert False, "request method must be set"
if request.path is None:
assert False, "request path must be set"
if request.host is None:
assert False, "request host must be set"
writer = self._stream.writer
writer.clear()
writer.write_bytes("%s %s HTTP/1.1\r\n" %
(request.method, request.path))
writer.write_bytes("Host: %s\r\n" % request.host)
for header_name, header_value in request.headers:
writer.write_bytes("%s: %s\r\n" % (header_name, header_value))
writer.write_bytes("\r\n")
if request.body is not None:
writer.write_bytes(request.body)
writer.flush()
def close(self):
"""Close this connection."""
self._stream.close()
class SocketTaskHandler(object):
'''
The SocketTask starts a tasklet that listens for incomming connections. Only one connection will be granted access.
When a client is connected, a second tasklet will be started. One tasklet is responsible for receiving messages, the other for sending messages.
The SocketTask can receive and send dicts: the dicts are serialized and deserialized using yaml, messages are seperated using '\n---\n'.
When the task receives a message, it will create a new tasklet and call the messageHandler, passing it the received data as a parameter.
'''
def __init__(self):
#self.__server_socket = Socket.new()
#self.__server_socket.set_reuse_address(1)
#self.__server_socket.bind(('', port))
#self.__server_socket.listen()
self.__client_socket = None
self.__receiving_tasklet = None
self.__sending_tasklet = None
def setMessageHandler(self, messageHandler):
'''
Set the callback that will be called when a message is received. The callback will be passed a dictionary.
'''
self.__messageHandler = messageHandler
def start(self, client_socket):
''' Start the task: start one tasklet listing for incomming connections. '''
#self.__receiving_tasklet = Tasklet.new(self.__serve)()
self.__client_socket = client_socket
self.__stream = BufferedStream(self.__client_socket)
q.logger.log("[SocketTaskHandler] Client connected.", 3)
self.__sending_tasklet = Tasklet.new(self.__send)()
self.__receiving_tasklet = Tasklet.new(self.__receive())()
def sendData(self, data):
'''
Send data over the socket.
@param data: the data to send
@raise Exception: if no client is connected
'''
if self.__sending_tasklet == None:
q.logger.log("[SocketTaskHandler] No connection to client, can't send the message.", 1)
raise Exception("No connection to client, can't send the message.")
else:
MSG_SOCKET_SEND.send(self.__sending_tasklet)(data)
def stop(self):
self.__stream and self.__stream.close()
def __receive(self):
# Called in the receiving tasklet
reader = self.__stream.reader
try:
buffer = ""
processData = True
while True:
line = reader.read_line()
if line <> '---':
buffer += line + "\n"
else:
try:
data = yaml.load(buffer)
processData = True
if data == 'ping':
processData = False
self.sendData('pong')
except yaml.parser.ParserError:
q.logger.log("[SocketTaskHandler] Received bad formatted data: " + str(buffer), 3)
else:
if processData:
q.logger.log("[SocketTaskHandler] Received data: " + str(data), 5)
Tasklet.new(self.__messageHandler)(data, self)
buffer = ""
except EOFError:
q.logger.log("[SocketTaskHandler] Client disconnected.", 3)
MSG_SOCKET_CLOSE.send(self.__sending_tasklet)()
self.__stream.close()
def __send(self):
# Started in the sending tasklet
writer = self.__stream.writer
for msg, args, kwargs in Tasklet.receive():
if msg.match(MSG_SOCKET_SEND):
message = self.__yaml_message(args[0])
q.logger.log("[SocketTaskHandler] Sending message: " + message, 5)
writer.write_bytes(message)
writer.flush()
elif msg.match(MSG_SOCKET_CLOSE):
return
def __yaml_message(self, dict):
return yaml.dump(dict) + "\n---\n"
class HTTPConnection(object):
"""A HTTP 1.1 Client.
Usage::
#create an instance of this class and connect to a webserver using the connect method:
cnn = HTTPConnection()
cnn.connect(('www.google.com', 80))
#create a GET request using the get method:
request = cnn.get('/index.html')
#finally perform the request to get a response:
response = cnn.perform(request)
#do something with the response:
print response.body
"""
log = logging.getLogger('HTTPConnection')
def __init__(self):
self.limit = None
def connect(self, endpoint):
"""Connect to the webserver at *endpoint*. *endpoint* is a tuple (<host>, <port>)."""
self._host = None
if type(endpoint) == type(()):
try:
self._host = endpoint[0]
except Exception:
pass
self._stream = BufferedStream(Connector.connect(endpoint), read_buffer_size = 1024 * 8, write_buffer_size = 1024 * 4)
def set_limit(self, limit):
self.limit = limit
def receive(self):
"""Receive the next :class:`HTTPResponse` from the connection."""
try:
return self._receive()
except TaskletExit:
raise
except EOFError:
raise HTTPError("EOF while reading response")
except HTTPError:
raise
except TimeoutError:
raise
except Exception:
self.log.exception('')
raise HTTPError("Exception while reading response")
def _receive(self):
response = HTTPResponse()
with self._stream.get_reader() as reader:
lines = reader.read_lines()
#parse status line
response.status = lines.next()
#rest of response headers
for line in lines:
if not line: break
key, value = line.split(': ')
response.add_header(key, value)
chunks = []
if response.status_code != 204:
#read data
transfer_encoding = response.get_header('Transfer-Encoding', None)
try:
content_length = int(response.get_header('Content-Length'))
if self.limit is not None and content_length > self.limit:
raise HTTPError("Response is too long")
except Exception:
content_length = None
#TODO better support large data, e.g. iterator instead of append all data to chunks
if transfer_encoding == 'chunked':
while True:
chunk_line = reader.read_line()
chunk_size = int(chunk_line.split(';')[0], 16)
if chunk_size > 0:
data = reader.read_bytes(chunk_size)
reader.read_line() #chunk is always followed by a single empty line
chunks.append(data)
else:
reader.read_line() #chunk is always followed by a single empty line
break
elif content_length is not None:
while content_length > 0:
n = min(CHUNK_SIZE, content_length)
data = reader.read_bytes(n)
chunks.append(data)
content_length -= len(data)
else:
content_length = 0
while True:
try:
data = reader.read_bytes_available()
except EOFError:
break
chunks.append(data)
content_length += len(data)
if self.limit is not None and content_length > self.limit:
raise HTTPError("Response is too long")
response.iter = chunks
return response
def get(self, path, host = None):
"""Returns a new :class:`HTTPRequest` with request.method = 'GET' and request.path = *path*.
request.host will be set to the host used in :func:`connect`, or optionally you can specify a
specific *host* just for this request.
"""
request = HTTPRequest()
request.method = 'GET'
request.path = path
request.host = host or self._host
return request
def post(self, path, body = None, host = None):
"""Returns a new :class:`HTTPRequest` with request.method = 'POST' and request.path = *path*.
request.host will be set to the host used in :func:`connect`, or optionally you can specify a
specific *host* just for this request.
*body* is an optional string containing the data to post to the server.
"""
request = HTTPRequest()
request.method = 'POST'
request.path = path
request.host = host or self._host
if body is not None:
request.body = body
return request
def perform(self, request):
"""Sends the *request* and waits for and returns the :class:`HTTPResult`."""
self.send(request)
return self.receive()
def send(self, request):
"""Sends the *request* on this connection."""
if request.method is None:
assert False, "request method must be set"
if request.path is None:
assert False, "request path must be set"
if request.host is None:
assert False, "request host must be set"
with self._stream.get_writer() as writer:
writer.clear()
writer.write_bytes("%s %s HTTP/1.1\r\n" % (request.method, request.path))
writer.write_bytes("Host: %s\r\n" % request.host)
for header_name, header_value in request.headers:
writer.write_bytes("%s: %s\r\n" % (header_name, header_value))
writer.write_bytes("\r\n")
if request.body is not None:
writer.write_bytes(request.body)
writer.flush()
def close(self):
"""Close this connection."""
self._stream.close()
def testTextProtocol(self):
from concurrence.io import Socket, BufferedStream
from concurrence.memcache.protocol import MemcacheProtocol
socket = Socket.connect((MEMCACHE_IP, 11211))
stream = BufferedStream(socket)
writer = stream.writer
reader = stream.reader
try:
protocol = MemcacheProtocol.create("textblaat")
self.fail("expected error")
except MemcacheError:
pass
protocol = MemcacheProtocol.create("text")
self.assertTrue(protocol is MemcacheProtocol.create(protocol))
protocol.set_codec("raw")
protocol.write_set(writer, 'hello', 'world', 0, 0)
writer.flush()
self.assertEquals((MemcacheResult.STORED, None),
protocol.read_set(reader))
N = 100
for i in range(N):
protocol.write_set(writer, 'test%d' % i, 'hello world %d' % i, 0,
0)
writer.flush()
self.assertEquals((MemcacheResult.STORED, None),
protocol.read_set(reader))
#single get
for i in range(N):
protocol.write_get(writer, ['test%d' % i])
writer.flush()
result = protocol.read_get(reader)
self.assertEquals((MemcacheResult.OK, {
'test%d' % i: 'hello world %d' % i
}), result)
#multi get
for i in range(0, N, 10):
keys = ['test%d' % x for x in range(i, i + 10)]
protocol.write_get(writer, keys)
writer.flush()
result, values = protocol.read_get(reader)
self.assertEquals(MemcacheResult.OK, result)
self.assertEquals(10, len(values))
#multi get pipeline, e.g. write N gets, but don't read out the results yet
for i in range(0, N, 10):
keys = ['test%d' % x for x in range(i, i + 10)]
protocol.write_get(writer, keys)
writer.flush()
#now read the results
for i in range(0, N, 10):
result, values = protocol.read_get(reader)
self.assertEquals(10, len(values))
self.assertTrue(('test%d' % i) in values)
#pipelined multiget with same set of keys
protocol.write_get(writer, [
'test2', 'test8', 'test9', 'test11', 'test23', 'test24', 'test29',
'test31', 'test34'
])
writer.flush()
protocol.write_get(writer, [
'test2', 'test8', 'test9', 'test11', 'test23', 'test24', 'test29',
'test31', 'test34'
])
writer.flush()
result1 = protocol.read_get(reader)
result2 = protocol.read_get(reader)
self.assertEquals(result1, result2)
class HTTPConnection(object):
def __init__(self, server, client_socket):
self._server = server
self._stream = BufferedStream(client_socket)
#print 'new con'
def _write_response(self, version, status, headers, response):
if version == 'HTTP/1.0':
chunked = False
else:
chunked = True
headers.append(('Date', rfc822.formatdate()))
headers.append(('Server', SERVER_ID))
if chunked:
headers.append(('Transfer-Encoding', 'chunked'))
else:
headers.append(('Content-length', str(len(response))))
response = ''.join(response)
with self._stream.get_writer() as writer:
writer.clear()
writer.write_bytes("%s %s\r\n" % (version, status))
writer.write_bytes('\r\n'.join(["%s: %s" % (k, v) for k, v in headers]))
writer.write_bytes("\r\n\r\n")
if chunked:
for chunk in response:
writer.write_bytes("%x;\r\n" % len(chunk))
writer.write_bytes(chunk)
writer.write_bytes("\r\n")
writer.write_bytes("0\r\n\r\n")
else:
writer.write_bytes(response)
writer.flush()
def _read_request(self):
with self._stream.get_reader() as reader:
try:
reader.fill() #initial fill
except EOFError:
return None
parser = HTTPParser(reader.buffer)
while True:
#parse the buffer
if parser.parse():
break #ok
else:
#need more data from socket, could not parse request with data currently in buffer
reader.append()
return WSGIRequest(parser.environ)
def _handle_request(self):
request = self._read_request()
if request == None:
return
response = self._server.handle_request(request)
self._write_response(request.version, request.response_status, request.response_headers, response)
if request.version == 'HTTP/1.0':
self._close()
else:
self._stream._stream.readable.notify(self.handle, 10)
def _close(self):
self._stream.close()
def handle(self, has_timedout = False):
if has_timedout:
self._stream.close()
else:
Tasklet.defer(self._handle_request)
def __init__(self, client_stream):
self._client_stream = BufferedStream(client_stream)
self._object_reader = ObjectReader(self._client_stream.reader)
self._object_writer = ObjectWriter(self._client_stream.writer)
def connect(self, endpoint):
self._stream = BufferedStream(Connector.connect(endpoint))
self._message_writer_task = Tasklet.new(self._message_writer)()
self._message_reader_task = Tasklet.new(self._message_reader)()
def __init__(self, server, client_socket):
self._server = server
self._stream = BufferedStream(client_socket)
def connect(self):
self._stream = BufferedStream(Socket.connect(self._address))
