class HTTPConnection(object):
def __init__(self, server, client_socket):
self._server = server
self._stream = BufferedStream(client_socket)
#print 'new con'
def _write_response(self, version, status, headers, response):
if version == 'HTTP/1.0':
chunked = False
else:
chunked = True
headers.append(('Date', rfc822.formatdate()))
headers.append(('Server', SERVER_ID))
if chunked:
headers.append(('Transfer-Encoding', 'chunked'))
else:
headers.append(('Content-length', str(len(response))))
response = ''.join(response)
with self._stream.get_writer() as writer:
writer.clear()
writer.write_bytes("%s %s\r\n" % (version, status))
writer.write_bytes('\r\n'.join(["%s: %s" % (k, v) for k, v in headers]))
writer.write_bytes("\r\n\r\n")
if chunked:
for chunk in response:
writer.write_bytes("%x;\r\n" % len(chunk))
writer.write_bytes(chunk)
writer.write_bytes("\r\n")
writer.write_bytes("0\r\n\r\n")
else:
writer.write_bytes(response)
writer.flush()
def _read_request(self):
with self._stream.get_reader() as reader:
try:
reader.fill() #initial fill
except EOFError:
return None
parser = HTTPParser(reader.buffer)
while True:
#parse the buffer
if parser.parse():
break #ok
else:
#need more data from socket, could not parse request with data currently in buffer
reader.append()
return WSGIRequest(parser.environ)
def _handle_request(self):
request = self._read_request()
if request == None:
return
response = self._server.handle_request(request)
self._write_response(request.version, request.response_status, request.response_headers, response)
if request.version == 'HTTP/1.0':
self._close()
else:
self._stream._stream.readable.notify(self.handle, 10)
def _close(self):
self._stream.close()
def handle(self, has_timedout = False):
if has_timedout:
self._stream.close()
else:
Tasklet.defer(self._handle_request)
class MemcacheConnection(object):
log = logging.getLogger("MemcacheConnection")
_tasklet_pool = TaskletPool(worker_timeout=2.0,
worker_timeout_relative=False)
def __init__(self, address, protocol="text", codec="default"):
self._address = address
self._stream = None
self._read_queue = DeferredQueue(self._tasklet_pool.defer)
self._write_queue = DeferredQueue(self._tasklet_pool.defer)
self._protocol = MemcacheProtocol.create(protocol)
self._protocol.set_codec(MemcacheCodec.create(codec))
def connect(self):
self._stream = BufferedStream(Socket.connect(self._address))
def is_connected(self):
return self._stream is not None
def _defer_commands(self, cmds, result_channel):
def _read_results():
protocol = self._protocol
with self._stream.get_reader() as reader:
for cmd, args, error_value in cmds:
try:
result = protocol.read(cmd, reader)
result_channel.send(result)
except TaskletExit:
raise
except:
self.log.exception("read error in defer_commands")
result_channel.send(
(MemcacheResult.ERROR, error_value))
#end _read_commands
def _write_commands():
protocol = self._protocol
try:
if not self.is_connected():
self.connect()
except TaskletExit:
raise
except:
self.log.exception("connect error in defer_commands")
for _, _, error_value in cmds:
result_channel.send((MemcacheResult.ERROR, error_value))
return
with self._stream.get_writer() as writer:
for cmd, args, error_value in cmds:
try:
protocol.write(cmd, writer, args)
except TaskletExit:
raise
except:
self.log.exception("write error in defer_commands")
result_channel.send(
(MemcacheResult.ERROR, error_value))
writer.flush()
self._read_queue.defer(_read_results)
#end _write_commands
self._write_queue.defer(_write_commands)
def _defer_command(self, cmd, args, result_channel, error_value=None):
self._defer_commands([(cmd, args, error_value)], result_channel)
def _do_command(self, cmd, args, error_value=None):
result_channel = ResultChannel()
self._defer_command(cmd, args, result_channel, error_value)
try:
return result_channel.receive()
except TimeoutError:
return MemcacheResult.TIMEOUT, error_value
def close(self):
if self.is_connected():
self._stream.close()
self._stream = None
def __setitem__(self, key, data):
self.set(key, data)
def __getitem__(self, key):
return self.get(key)
def delete(self, key, expiration=0):
return self._do_command("delete", (key, expiration))[0]
def set(self, key, data, expiration=0, flags=0):
return self._do_command("set", (key, data, expiration, flags))[0]
def add(self, key, data, expiration=0, flags=0):
return self._do_command("add", (key, data, expiration, flags))[0]
def replace(self, key, data, expiration=0, flags=0):
return self._do_command("replace", (key, data, expiration, flags))[0]
def append(self, key, data, expiration=0, flags=0):
return self._do_command("append", (key, data, expiration, flags))[0]
def prepend(self, key, data, expiration=0, flags=0):
return self._do_command("prepend", (key, data, expiration, flags))[0]
def cas(self, key, data, cas_unique, expiration=0, flags=0):
return self._do_command("cas",
(key, data, expiration, flags, cas_unique))[0]
def incr(self, key, increment):
return self._do_command("incr", (key, increment))
def decr(self, key, increment):
return self._do_command("decr", (key, increment))
def get(self, key, default=None):
_, values = self._do_command("get", ([key], ), {})
return values.get(key, default)
def getr(self, key, default=None):
result, values = self._do_command("get", ([key], ), {})
return result, values.get(key, default)
def gets(self, key, default=None):
result, values = self._do_command("gets", ([key], ), {})
value, cas_unique = values.get(key, (default, None))
return result, value, cas_unique
def get_multi(self, keys):
return self._do_command("get", (keys, ))
def gets_multi(self, keys):
return self._do_command("gets", (keys, ))
def version(self):
return self._do_command("version", ())
def batch(self):
return CommandBatch(self)
class HTTPConnection(object):
"""A HTTP 1.1 Client.
Usage::
#create an instance of this class and connect to a webserver using the connect method:
cnn = HTTPConnection()
cnn.connect(('www.google.com', 80))
#create a GET request using the get method:
request = cnn.get('/index.html')
#finally perform the request to get a response:
response = cnn.perform(request)
#do something with the response:
print response.body
"""
log = logging.getLogger('HTTPConnection')
def __init__(self):
self.limit = None
def connect(self, endpoint):
"""Connect to the webserver at *endpoint*. *endpoint* is a tuple (<host>, <port>)."""
self._host = None
if type(endpoint) == type(()):
try:
self._host = endpoint[0]
except Exception:
pass
self._stream = BufferedStream(Connector.connect(endpoint), read_buffer_size = 1024 * 8, write_buffer_size = 1024 * 4)
def set_limit(self, limit):
self.limit = limit
def receive(self):
"""Receive the next :class:`HTTPResponse` from the connection."""
try:
return self._receive()
except TaskletExit:
raise
except EOFError:
raise HTTPError("EOF while reading response")
except HTTPError:
raise
except TimeoutError:
raise
except Exception:
self.log.exception('')
raise HTTPError("Exception while reading response")
def _receive(self):
response = HTTPResponse()
with self._stream.get_reader() as reader:
lines = reader.read_lines()
#parse status line
response.status = lines.next()
#rest of response headers
for line in lines:
if not line: break
key, value = line.split(': ')
response.add_header(key, value)
chunks = []
if response.status_code != 204:
#read data
transfer_encoding = response.get_header('Transfer-Encoding', None)
try:
content_length = int(response.get_header('Content-Length'))
if self.limit is not None and content_length > self.limit:
raise HTTPError("Response is too long")
except Exception:
content_length = None
#TODO better support large data, e.g. iterator instead of append all data to chunks
if transfer_encoding == 'chunked':
while True:
chunk_line = reader.read_line()
chunk_size = int(chunk_line.split(';')[0], 16)
if chunk_size > 0:
data = reader.read_bytes(chunk_size)
reader.read_line() #chunk is always followed by a single empty line
chunks.append(data)
else:
reader.read_line() #chunk is always followed by a single empty line
break
elif content_length is not None:
while content_length > 0:
n = min(CHUNK_SIZE, content_length)
data = reader.read_bytes(n)
chunks.append(data)
content_length -= len(data)
else:
content_length = 0
while True:
try:
data = reader.read_bytes_available()
except EOFError:
break
chunks.append(data)
content_length += len(data)
if self.limit is not None and content_length > self.limit:
raise HTTPError("Response is too long")
response.iter = chunks
return response
def get(self, path, host = None):
"""Returns a new :class:`HTTPRequest` with request.method = 'GET' and request.path = *path*.
request.host will be set to the host used in :func:`connect`, or optionally you can specify a
specific *host* just for this request.
"""
request = HTTPRequest()
request.method = 'GET'
request.path = path
request.host = host or self._host
return request
def post(self, path, body = None, host = None):
"""Returns a new :class:`HTTPRequest` with request.method = 'POST' and request.path = *path*.
request.host will be set to the host used in :func:`connect`, or optionally you can specify a
specific *host* just for this request.
*body* is an optional string containing the data to post to the server.
"""
request = HTTPRequest()
request.method = 'POST'
request.path = path
request.host = host or self._host
if body is not None:
request.body = body
return request
def perform(self, request):
"""Sends the *request* and waits for and returns the :class:`HTTPResult`."""
self.send(request)
return self.receive()
def send(self, request):
"""Sends the *request* on this connection."""
if request.method is None:
assert False, "request method must be set"
if request.path is None:
assert False, "request path must be set"
if request.host is None:
assert False, "request host must be set"
with self._stream.get_writer() as writer:
writer.clear()
writer.write_bytes("%s %s HTTP/1.1\r\n" % (request.method, request.path))
writer.write_bytes("Host: %s\r\n" % request.host)
for header_name, header_value in request.headers:
writer.write_bytes("%s: %s\r\n" % (header_name, header_value))
writer.write_bytes("\r\n")
if request.body is not None:
writer.write_bytes(request.body)
writer.flush()
def close(self):
"""Close this connection."""
self._stream.close()
class HTTPConnection(object):
"""A HTTP 1.1 Client.
Usage::
#create an instance of this class and connect to a webserver using the connect method:
cnn = HTTPConnection()
cnn.connect(('www.google.com', 80))
#create a GET request using the get method:
request = cnn.get('/index.html')
#finally perform the request to get a response:
response = cnn.perform(request)
#do something with the response:
print response.body
"""
log = logging.getLogger('HTTPConnection')
def connect(self, endpoint):
"""Connect to the webserver at *endpoint*. *endpoint* is a tuple (<host>, <port>)."""
self._host = None
if type(endpoint) == type(()):
try:
self._host = endpoint[0]
except:
pass
self._stream = BufferedStream(Connector.connect(endpoint),
read_buffer_size=1024 * 8,
write_buffer_size=1024 * 4)
def receive(self):
"""Receive the next :class:`HTTPResponse` from the connection."""
try:
return self._receive()
except TaskletExit:
raise
except EOFError:
raise HTTPError("EOF while reading response")
except Exception:
self.log.exception('')
raise HTTPError("Exception while reading response")
def _receive(self):
response = HTTPResponse()
with self._stream.get_reader() as reader:
lines = reader.read_lines()
#parse status line
response.status = lines.next()
#rest of response headers
for line in lines:
if not line: break
key, value = line.split(': ')
response.add_header(key, value)
#read data
transfer_encoding = response.get_header('Transfer-Encoding', None)
try:
content_length = int(response.get_header('Content-Length'))
except:
content_length = None
#TODO better support large data, e.g. iterator instead of append all data to chunks
chunks = []
if transfer_encoding == 'chunked':
while True:
chunk_line = reader.read_line()
chunk_size = int(chunk_line.split(';')[0], 16)
if chunk_size > 0:
data = reader.read_bytes(chunk_size)
reader.read_line(
) #chunk is always followed by a single empty line
chunks.append(data)
else:
reader.read_line(
) #chunk is always followed by a single empty line
break
elif content_length is not None:
while content_length > 0:
n = min(CHUNK_SIZE, content_length)
data = reader.read_bytes(n)
chunks.append(data)
content_length -= len(data)
else:
assert False, 'TODO'
response.iter = chunks
return response
def get(self, path, host=None):
"""Returns a new :class:`HTTPRequest` with request.method = 'GET' and request.path = *path*.
request.host will be set to the host used in :func:`connect`, or optionally you can specify a
specific *host* just for this request.
"""
request = HTTPRequest()
request.method = 'GET'
request.path = path
request.host = host or self._host
return request
def post(self, path, body=None, host=None):
"""Returns a new :class:`HTTPRequest` with request.method = 'POST' and request.path = *path*.
request.host will be set to the host used in :func:`connect`, or optionally you can specify a
specific *host* just for this request.
*body* is an optional string containing the data to post to the server.
"""
request = HTTPRequest()
request.method = 'POST'
request.path = path
request.host = host or self._host
if body is not None:
request.body = body
return request
def perform(self, request):
"""Sends the *request* and waits for and returns the :class:`HTTPResult`."""
self.send(request)
return self.receive()
def send(self, request):
"""Sends the *request* on this connection."""
if request.method is None:
assert False, "request method must be set"
if request.path is None:
assert False, "request path must be set"
if request.host is None:
assert False, "request host must be set"
with self._stream.get_writer() as writer:
writer.clear()
writer.write_bytes("%s %s HTTP/1.1\r\n" %
(request.method, request.path))
writer.write_bytes("Host: %s\r\n" % request.host)
for header_name, header_value in request.headers:
writer.write_bytes("%s: %s\r\n" % (header_name, header_value))
writer.write_bytes("\r\n")
if request.body is not None:
writer.write_bytes(request.body)
writer.flush()
def close(self):
"""Close this connection."""
self._stream.close()
class MemcacheConnection(object):
log = logging.getLogger("MemcacheConnection")
_tasklet_pool = TaskletPool(worker_timeout = 2.0,
worker_timeout_relative = False)
def __init__(self, address, protocol = "text", codec = "default"):
self._address = address
self._stream = None
self._read_queue = DeferredQueue(self._tasklet_pool.defer)
self._write_queue = DeferredQueue(self._tasklet_pool.defer)
self._protocol = MemcacheProtocol.create(protocol)
self._protocol.set_codec(MemcacheCodec.create(codec))
def connect(self):
self._stream = BufferedStream(Socket.connect(self._address))
def is_connected(self):
return self._stream is not None
def _defer_commands(self, cmds, result_channel):
def _read_results():
protocol = self._protocol
with self._stream.get_reader() as reader:
for cmd, args, error_value in cmds:
try:
result = protocol.read(cmd, reader)
result_channel.send(result)
except TaskletExit:
raise
except:
self.log.exception("read error in defer_commands")
result_channel.send((MemcacheResult.ERROR, error_value))
#end _read_commands
def _write_commands():
protocol = self._protocol
try:
if not self.is_connected():
self.connect()
except TaskletExit:
raise
except:
self.log.exception("connect error in defer_commands")
for _, _, error_value in cmds:
result_channel.send((MemcacheResult.ERROR, error_value))
return
with self._stream.get_writer() as writer:
for cmd, args, error_value in cmds:
try:
protocol.write(cmd, writer, args)
except TaskletExit:
raise
except:
self.log.exception("write error in defer_commands")
result_channel.send((MemcacheResult.ERROR, error_value))
writer.flush()
self._read_queue.defer(_read_results)
#end _write_commands
self._write_queue.defer(_write_commands)
def _defer_command(self, cmd, args, result_channel, error_value = None):
self._defer_commands([(cmd, args, error_value)], result_channel)
def _do_command(self, cmd, args, error_value = None):
result_channel = ResultChannel()
self._defer_command(cmd, args, result_channel, error_value)
try:
return result_channel.receive()
except TimeoutError:
return MemcacheResult.TIMEOUT, error_value
def close(self):
if self.is_connected():
self._stream.close()
self._stream = None
def __setitem__(self, key, data):
self.set(key, data)
def __getitem__(self, key):
return self.get(key)
def delete(self, key, expiration = 0):
return self._do_command("delete", (key, expiration))[0]
def set(self, key, data, expiration = 0, flags = 0):
return self._do_command("set", (key, data, expiration, flags))[0]
def add(self, key, data, expiration = 0, flags = 0):
return self._do_command("add", (key, data, expiration, flags))[0]
def replace(self, key, data, expiration = 0, flags = 0):
return self._do_command("replace", (key, data, expiration, flags))[0]
def append(self, key, data, expiration = 0, flags = 0):
return self._do_command("append", (key, data, expiration, flags))[0]
def prepend(self, key, data, expiration = 0, flags = 0):
return self._do_command("prepend", (key, data, expiration, flags))[0]
def cas(self, key, data, cas_unique, expiration = 0, flags = 0):
return self._do_command("cas", (key, data, expiration, flags, cas_unique))[0]
def incr(self, key, increment):
return self._do_command("incr", (key, increment))
def decr(self, key, increment):
return self._do_command("decr", (key, increment))
def get(self, key, default = None):
_, values = self._do_command("get", ([key], ), {})
return values.get(key, default)
def getr(self, key, default = None):
result, values = self._do_command("get", ([key], ), {})
return result, values.get(key, default)
def gets(self, key, default = None):
result, values = self._do_command("gets", ([key], ), {})
value, cas_unique = values.get(key, (default, None))
return result, value, cas_unique
def get_multi(self, keys):
return self._do_command("get", (keys, ))
def gets_multi(self, keys):
return self._do_command("gets", (keys, ))
def version(self):
return self._do_command("version", ())
def batch(self):
return CommandBatch(self)