class HTTPConnection(object): def __init__(self, server, client_socket): self._server = server self._stream = BufferedStream(client_socket) #print 'new con' def _write_response(self, version, status, headers, response): if version == 'HTTP/1.0': chunked = False else: chunked = True headers.append(('Date', rfc822.formatdate())) headers.append(('Server', SERVER_ID)) if chunked: headers.append(('Transfer-Encoding', 'chunked')) else: headers.append(('Content-length', str(len(response)))) response = ''.join(response) with self._stream.get_writer() as writer: writer.clear() writer.write_bytes("%s %s\r\n" % (version, status)) writer.write_bytes('\r\n'.join(["%s: %s" % (k, v) for k, v in headers])) writer.write_bytes("\r\n\r\n") if chunked: for chunk in response: writer.write_bytes("%x;\r\n" % len(chunk)) writer.write_bytes(chunk) writer.write_bytes("\r\n") writer.write_bytes("0\r\n\r\n") else: writer.write_bytes(response) writer.flush() def _read_request(self): with self._stream.get_reader() as reader: try: reader.fill() #initial fill except EOFError: return None parser = HTTPParser(reader.buffer) while True: #parse the buffer if parser.parse(): break #ok else: #need more data from socket, could not parse request with data currently in buffer reader.append() return WSGIRequest(parser.environ) def _handle_request(self): request = self._read_request() if request == None: return response = self._server.handle_request(request) self._write_response(request.version, request.response_status, request.response_headers, response) if request.version == 'HTTP/1.0': self._close() else: self._stream._stream.readable.notify(self.handle, 10) def _close(self): self._stream.close() def handle(self, has_timedout = False): if has_timedout: self._stream.close() else: Tasklet.defer(self._handle_request)
class MemcacheConnection(object): log = logging.getLogger("MemcacheConnection") _tasklet_pool = TaskletPool(worker_timeout=2.0, worker_timeout_relative=False) def __init__(self, address, protocol="text", codec="default"): self._address = address self._stream = None self._read_queue = DeferredQueue(self._tasklet_pool.defer) self._write_queue = DeferredQueue(self._tasklet_pool.defer) self._protocol = MemcacheProtocol.create(protocol) self._protocol.set_codec(MemcacheCodec.create(codec)) def connect(self): self._stream = BufferedStream(Socket.connect(self._address)) def is_connected(self): return self._stream is not None def _defer_commands(self, cmds, result_channel): def _read_results(): protocol = self._protocol with self._stream.get_reader() as reader: for cmd, args, error_value in cmds: try: result = protocol.read(cmd, reader) result_channel.send(result) except TaskletExit: raise except: self.log.exception("read error in defer_commands") result_channel.send( (MemcacheResult.ERROR, error_value)) #end _read_commands def _write_commands(): protocol = self._protocol try: if not self.is_connected(): self.connect() except TaskletExit: raise except: self.log.exception("connect error in defer_commands") for _, _, error_value in cmds: result_channel.send((MemcacheResult.ERROR, error_value)) return with self._stream.get_writer() as writer: for cmd, args, error_value in cmds: try: protocol.write(cmd, writer, args) except TaskletExit: raise except: self.log.exception("write error in defer_commands") result_channel.send( (MemcacheResult.ERROR, error_value)) writer.flush() self._read_queue.defer(_read_results) #end _write_commands self._write_queue.defer(_write_commands) def _defer_command(self, cmd, args, result_channel, error_value=None): self._defer_commands([(cmd, args, error_value)], result_channel) def _do_command(self, cmd, args, error_value=None): result_channel = ResultChannel() self._defer_command(cmd, args, result_channel, error_value) try: return result_channel.receive() except TimeoutError: return MemcacheResult.TIMEOUT, error_value def close(self): if self.is_connected(): self._stream.close() self._stream = None def __setitem__(self, key, data): self.set(key, data) def __getitem__(self, key): return self.get(key) def delete(self, key, expiration=0): return self._do_command("delete", (key, expiration))[0] def set(self, key, data, expiration=0, flags=0): return self._do_command("set", (key, data, expiration, flags))[0] def add(self, key, data, expiration=0, flags=0): return self._do_command("add", (key, data, expiration, flags))[0] def replace(self, key, data, expiration=0, flags=0): return self._do_command("replace", (key, data, expiration, flags))[0] def append(self, key, data, expiration=0, flags=0): return self._do_command("append", (key, data, expiration, flags))[0] def prepend(self, key, data, expiration=0, flags=0): return self._do_command("prepend", (key, data, expiration, flags))[0] def cas(self, key, data, cas_unique, expiration=0, flags=0): return self._do_command("cas", (key, data, expiration, flags, cas_unique))[0] def incr(self, key, increment): return self._do_command("incr", (key, increment)) def decr(self, key, increment): return self._do_command("decr", (key, increment)) def get(self, key, default=None): _, values = self._do_command("get", ([key], ), {}) return values.get(key, default) def getr(self, key, default=None): result, values = self._do_command("get", ([key], ), {}) return result, values.get(key, default) def gets(self, key, default=None): result, values = self._do_command("gets", ([key], ), {}) value, cas_unique = values.get(key, (default, None)) return result, value, cas_unique def get_multi(self, keys): return self._do_command("get", (keys, )) def gets_multi(self, keys): return self._do_command("gets", (keys, )) def version(self): return self._do_command("version", ()) def batch(self): return CommandBatch(self)
class HTTPConnection(object): """A HTTP 1.1 Client. Usage:: #create an instance of this class and connect to a webserver using the connect method: cnn = HTTPConnection() cnn.connect(('www.google.com', 80)) #create a GET request using the get method: request = cnn.get('/index.html') #finally perform the request to get a response: response = cnn.perform(request) #do something with the response: print response.body """ log = logging.getLogger('HTTPConnection') def __init__(self): self.limit = None def connect(self, endpoint): """Connect to the webserver at *endpoint*. *endpoint* is a tuple (<host>, <port>).""" self._host = None if type(endpoint) == type(()): try: self._host = endpoint[0] except Exception: pass self._stream = BufferedStream(Connector.connect(endpoint), read_buffer_size = 1024 * 8, write_buffer_size = 1024 * 4) def set_limit(self, limit): self.limit = limit def receive(self): """Receive the next :class:`HTTPResponse` from the connection.""" try: return self._receive() except TaskletExit: raise except EOFError: raise HTTPError("EOF while reading response") except HTTPError: raise except TimeoutError: raise except Exception: self.log.exception('') raise HTTPError("Exception while reading response") def _receive(self): response = HTTPResponse() with self._stream.get_reader() as reader: lines = reader.read_lines() #parse status line response.status = lines.next() #rest of response headers for line in lines: if not line: break key, value = line.split(': ') response.add_header(key, value) chunks = [] if response.status_code != 204: #read data transfer_encoding = response.get_header('Transfer-Encoding', None) try: content_length = int(response.get_header('Content-Length')) if self.limit is not None and content_length > self.limit: raise HTTPError("Response is too long") except Exception: content_length = None #TODO better support large data, e.g. iterator instead of append all data to chunks if transfer_encoding == 'chunked': while True: chunk_line = reader.read_line() chunk_size = int(chunk_line.split(';')[0], 16) if chunk_size > 0: data = reader.read_bytes(chunk_size) reader.read_line() #chunk is always followed by a single empty line chunks.append(data) else: reader.read_line() #chunk is always followed by a single empty line break elif content_length is not None: while content_length > 0: n = min(CHUNK_SIZE, content_length) data = reader.read_bytes(n) chunks.append(data) content_length -= len(data) else: content_length = 0 while True: try: data = reader.read_bytes_available() except EOFError: break chunks.append(data) content_length += len(data) if self.limit is not None and content_length > self.limit: raise HTTPError("Response is too long") response.iter = chunks return response def get(self, path, host = None): """Returns a new :class:`HTTPRequest` with request.method = 'GET' and request.path = *path*. request.host will be set to the host used in :func:`connect`, or optionally you can specify a specific *host* just for this request. """ request = HTTPRequest() request.method = 'GET' request.path = path request.host = host or self._host return request def post(self, path, body = None, host = None): """Returns a new :class:`HTTPRequest` with request.method = 'POST' and request.path = *path*. request.host will be set to the host used in :func:`connect`, or optionally you can specify a specific *host* just for this request. *body* is an optional string containing the data to post to the server. """ request = HTTPRequest() request.method = 'POST' request.path = path request.host = host or self._host if body is not None: request.body = body return request def perform(self, request): """Sends the *request* and waits for and returns the :class:`HTTPResult`.""" self.send(request) return self.receive() def send(self, request): """Sends the *request* on this connection.""" if request.method is None: assert False, "request method must be set" if request.path is None: assert False, "request path must be set" if request.host is None: assert False, "request host must be set" with self._stream.get_writer() as writer: writer.clear() writer.write_bytes("%s %s HTTP/1.1\r\n" % (request.method, request.path)) writer.write_bytes("Host: %s\r\n" % request.host) for header_name, header_value in request.headers: writer.write_bytes("%s: %s\r\n" % (header_name, header_value)) writer.write_bytes("\r\n") if request.body is not None: writer.write_bytes(request.body) writer.flush() def close(self): """Close this connection.""" self._stream.close()
class HTTPConnection(object): """A HTTP 1.1 Client. Usage:: #create an instance of this class and connect to a webserver using the connect method: cnn = HTTPConnection() cnn.connect(('www.google.com', 80)) #create a GET request using the get method: request = cnn.get('/index.html') #finally perform the request to get a response: response = cnn.perform(request) #do something with the response: print response.body """ log = logging.getLogger('HTTPConnection') def connect(self, endpoint): """Connect to the webserver at *endpoint*. *endpoint* is a tuple (<host>, <port>).""" self._host = None if type(endpoint) == type(()): try: self._host = endpoint[0] except: pass self._stream = BufferedStream(Connector.connect(endpoint), read_buffer_size=1024 * 8, write_buffer_size=1024 * 4) def receive(self): """Receive the next :class:`HTTPResponse` from the connection.""" try: return self._receive() except TaskletExit: raise except EOFError: raise HTTPError("EOF while reading response") except Exception: self.log.exception('') raise HTTPError("Exception while reading response") def _receive(self): response = HTTPResponse() with self._stream.get_reader() as reader: lines = reader.read_lines() #parse status line response.status = lines.next() #rest of response headers for line in lines: if not line: break key, value = line.split(': ') response.add_header(key, value) #read data transfer_encoding = response.get_header('Transfer-Encoding', None) try: content_length = int(response.get_header('Content-Length')) except: content_length = None #TODO better support large data, e.g. iterator instead of append all data to chunks chunks = [] if transfer_encoding == 'chunked': while True: chunk_line = reader.read_line() chunk_size = int(chunk_line.split(';')[0], 16) if chunk_size > 0: data = reader.read_bytes(chunk_size) reader.read_line( ) #chunk is always followed by a single empty line chunks.append(data) else: reader.read_line( ) #chunk is always followed by a single empty line break elif content_length is not None: while content_length > 0: n = min(CHUNK_SIZE, content_length) data = reader.read_bytes(n) chunks.append(data) content_length -= len(data) else: assert False, 'TODO' response.iter = chunks return response def get(self, path, host=None): """Returns a new :class:`HTTPRequest` with request.method = 'GET' and request.path = *path*. request.host will be set to the host used in :func:`connect`, or optionally you can specify a specific *host* just for this request. """ request = HTTPRequest() request.method = 'GET' request.path = path request.host = host or self._host return request def post(self, path, body=None, host=None): """Returns a new :class:`HTTPRequest` with request.method = 'POST' and request.path = *path*. request.host will be set to the host used in :func:`connect`, or optionally you can specify a specific *host* just for this request. *body* is an optional string containing the data to post to the server. """ request = HTTPRequest() request.method = 'POST' request.path = path request.host = host or self._host if body is not None: request.body = body return request def perform(self, request): """Sends the *request* and waits for and returns the :class:`HTTPResult`.""" self.send(request) return self.receive() def send(self, request): """Sends the *request* on this connection.""" if request.method is None: assert False, "request method must be set" if request.path is None: assert False, "request path must be set" if request.host is None: assert False, "request host must be set" with self._stream.get_writer() as writer: writer.clear() writer.write_bytes("%s %s HTTP/1.1\r\n" % (request.method, request.path)) writer.write_bytes("Host: %s\r\n" % request.host) for header_name, header_value in request.headers: writer.write_bytes("%s: %s\r\n" % (header_name, header_value)) writer.write_bytes("\r\n") if request.body is not None: writer.write_bytes(request.body) writer.flush() def close(self): """Close this connection.""" self._stream.close()
class MemcacheConnection(object): log = logging.getLogger("MemcacheConnection") _tasklet_pool = TaskletPool(worker_timeout = 2.0, worker_timeout_relative = False) def __init__(self, address, protocol = "text", codec = "default"): self._address = address self._stream = None self._read_queue = DeferredQueue(self._tasklet_pool.defer) self._write_queue = DeferredQueue(self._tasklet_pool.defer) self._protocol = MemcacheProtocol.create(protocol) self._protocol.set_codec(MemcacheCodec.create(codec)) def connect(self): self._stream = BufferedStream(Socket.connect(self._address)) def is_connected(self): return self._stream is not None def _defer_commands(self, cmds, result_channel): def _read_results(): protocol = self._protocol with self._stream.get_reader() as reader: for cmd, args, error_value in cmds: try: result = protocol.read(cmd, reader) result_channel.send(result) except TaskletExit: raise except: self.log.exception("read error in defer_commands") result_channel.send((MemcacheResult.ERROR, error_value)) #end _read_commands def _write_commands(): protocol = self._protocol try: if not self.is_connected(): self.connect() except TaskletExit: raise except: self.log.exception("connect error in defer_commands") for _, _, error_value in cmds: result_channel.send((MemcacheResult.ERROR, error_value)) return with self._stream.get_writer() as writer: for cmd, args, error_value in cmds: try: protocol.write(cmd, writer, args) except TaskletExit: raise except: self.log.exception("write error in defer_commands") result_channel.send((MemcacheResult.ERROR, error_value)) writer.flush() self._read_queue.defer(_read_results) #end _write_commands self._write_queue.defer(_write_commands) def _defer_command(self, cmd, args, result_channel, error_value = None): self._defer_commands([(cmd, args, error_value)], result_channel) def _do_command(self, cmd, args, error_value = None): result_channel = ResultChannel() self._defer_command(cmd, args, result_channel, error_value) try: return result_channel.receive() except TimeoutError: return MemcacheResult.TIMEOUT, error_value def close(self): if self.is_connected(): self._stream.close() self._stream = None def __setitem__(self, key, data): self.set(key, data) def __getitem__(self, key): return self.get(key) def delete(self, key, expiration = 0): return self._do_command("delete", (key, expiration))[0] def set(self, key, data, expiration = 0, flags = 0): return self._do_command("set", (key, data, expiration, flags))[0] def add(self, key, data, expiration = 0, flags = 0): return self._do_command("add", (key, data, expiration, flags))[0] def replace(self, key, data, expiration = 0, flags = 0): return self._do_command("replace", (key, data, expiration, flags))[0] def append(self, key, data, expiration = 0, flags = 0): return self._do_command("append", (key, data, expiration, flags))[0] def prepend(self, key, data, expiration = 0, flags = 0): return self._do_command("prepend", (key, data, expiration, flags))[0] def cas(self, key, data, cas_unique, expiration = 0, flags = 0): return self._do_command("cas", (key, data, expiration, flags, cas_unique))[0] def incr(self, key, increment): return self._do_command("incr", (key, increment)) def decr(self, key, increment): return self._do_command("decr", (key, increment)) def get(self, key, default = None): _, values = self._do_command("get", ([key], ), {}) return values.get(key, default) def getr(self, key, default = None): result, values = self._do_command("get", ([key], ), {}) return result, values.get(key, default) def gets(self, key, default = None): result, values = self._do_command("gets", ([key], ), {}) value, cas_unique = values.get(key, (default, None)) return result, value, cas_unique def get_multi(self, keys): return self._do_command("get", (keys, )) def gets_multi(self, keys): return self._do_command("gets", (keys, )) def version(self): return self._do_command("version", ()) def batch(self): return CommandBatch(self)