Esempio n. 1
0
class HTTPConnection(object):

    def __init__(self, server, client_socket):
        self._server = server
        self._stream = BufferedStream(client_socket)
        #print 'new con'

    def _write_response(self, version, status, headers, response):

        if version == 'HTTP/1.0':
            chunked = False
        else:
            chunked = True

        headers.append(('Date', rfc822.formatdate()))
        headers.append(('Server', SERVER_ID))

        if chunked:
            headers.append(('Transfer-Encoding', 'chunked'))
        else:
            headers.append(('Content-length', str(len(response))))
            response = ''.join(response)

        with self._stream.get_writer() as writer:
            writer.clear()
            writer.write_bytes("%s %s\r\n" % (version, status))
            writer.write_bytes('\r\n'.join(["%s: %s" % (k, v) for k, v in headers]))
            writer.write_bytes("\r\n\r\n")

            if chunked:
                for chunk in response:
                    writer.write_bytes("%x;\r\n" % len(chunk))
                    writer.write_bytes(chunk)
                    writer.write_bytes("\r\n")
                writer.write_bytes("0\r\n\r\n")
            else:
                writer.write_bytes(response)

            writer.flush()

    def _read_request(self):

        with self._stream.get_reader() as reader:
            try:
                reader.fill() #initial fill
            except EOFError:
                return None

            parser = HTTPParser(reader.buffer)
            while True:
                #parse the buffer
                if parser.parse():
                    break #ok
                else:
                    #need more data from socket, could not parse request with data currently in buffer
                    reader.append() 
            return WSGIRequest(parser.environ)

    def _handle_request(self):
        request = self._read_request()
        if request == None:
           return
        
        response = self._server.handle_request(request)
        self._write_response(request.version, request.response_status, request.response_headers, response)  
        if request.version == 'HTTP/1.0':
            self._close()
        else:
            self._stream._stream.readable.notify(self.handle, 10)

    def _close(self):
        self._stream.close()

    def handle(self, has_timedout = False):
        if has_timedout:
            self._stream.close()
        else:
            Tasklet.defer(self._handle_request)
Esempio n. 2
0
class MemcacheConnection(object):
    log = logging.getLogger("MemcacheConnection")

    _tasklet_pool = TaskletPool(worker_timeout=2.0,
                                worker_timeout_relative=False)

    def __init__(self, address, protocol="text", codec="default"):

        self._address = address

        self._stream = None
        self._read_queue = DeferredQueue(self._tasklet_pool.defer)
        self._write_queue = DeferredQueue(self._tasklet_pool.defer)

        self._protocol = MemcacheProtocol.create(protocol)
        self._protocol.set_codec(MemcacheCodec.create(codec))

    def connect(self):
        self._stream = BufferedStream(Socket.connect(self._address))

    def is_connected(self):
        return self._stream is not None

    def _defer_commands(self, cmds, result_channel):
        def _read_results():
            protocol = self._protocol
            with self._stream.get_reader() as reader:
                for cmd, args, error_value in cmds:
                    try:
                        result = protocol.read(cmd, reader)
                        result_channel.send(result)
                    except TaskletExit:
                        raise
                    except:
                        self.log.exception("read error in defer_commands")
                        result_channel.send(
                            (MemcacheResult.ERROR, error_value))

        #end _read_commands
        def _write_commands():
            protocol = self._protocol
            try:
                if not self.is_connected():
                    self.connect()
            except TaskletExit:
                raise
            except:
                self.log.exception("connect error in defer_commands")
                for _, _, error_value in cmds:
                    result_channel.send((MemcacheResult.ERROR, error_value))
                return
            with self._stream.get_writer() as writer:
                for cmd, args, error_value in cmds:
                    try:
                        protocol.write(cmd, writer, args)
                    except TaskletExit:
                        raise
                    except:
                        self.log.exception("write error in defer_commands")
                        result_channel.send(
                            (MemcacheResult.ERROR, error_value))
                writer.flush()
            self._read_queue.defer(_read_results)

        #end _write_commands
        self._write_queue.defer(_write_commands)

    def _defer_command(self, cmd, args, result_channel, error_value=None):
        self._defer_commands([(cmd, args, error_value)], result_channel)

    def _do_command(self, cmd, args, error_value=None):
        result_channel = ResultChannel()
        self._defer_command(cmd, args, result_channel, error_value)
        try:
            return result_channel.receive()
        except TimeoutError:
            return MemcacheResult.TIMEOUT, error_value

    def close(self):
        if self.is_connected():
            self._stream.close()
            self._stream = None

    def __setitem__(self, key, data):
        self.set(key, data)

    def __getitem__(self, key):
        return self.get(key)

    def delete(self, key, expiration=0):
        return self._do_command("delete", (key, expiration))[0]

    def set(self, key, data, expiration=0, flags=0):
        return self._do_command("set", (key, data, expiration, flags))[0]

    def add(self, key, data, expiration=0, flags=0):
        return self._do_command("add", (key, data, expiration, flags))[0]

    def replace(self, key, data, expiration=0, flags=0):
        return self._do_command("replace", (key, data, expiration, flags))[0]

    def append(self, key, data, expiration=0, flags=0):
        return self._do_command("append", (key, data, expiration, flags))[0]

    def prepend(self, key, data, expiration=0, flags=0):
        return self._do_command("prepend", (key, data, expiration, flags))[0]

    def cas(self, key, data, cas_unique, expiration=0, flags=0):
        return self._do_command("cas",
                                (key, data, expiration, flags, cas_unique))[0]

    def incr(self, key, increment):
        return self._do_command("incr", (key, increment))

    def decr(self, key, increment):
        return self._do_command("decr", (key, increment))

    def get(self, key, default=None):
        _, values = self._do_command("get", ([key], ), {})
        return values.get(key, default)

    def getr(self, key, default=None):
        result, values = self._do_command("get", ([key], ), {})
        return result, values.get(key, default)

    def gets(self, key, default=None):
        result, values = self._do_command("gets", ([key], ), {})
        value, cas_unique = values.get(key, (default, None))
        return result, value, cas_unique

    def get_multi(self, keys):
        return self._do_command("get", (keys, ))

    def gets_multi(self, keys):
        return self._do_command("gets", (keys, ))

    def version(self):
        return self._do_command("version", ())

    def batch(self):
        return CommandBatch(self)
Esempio n. 3
0
class HTTPConnection(object):
    """A HTTP 1.1 Client.

    Usage::

        #create an instance of this class and connect to a webserver using the connect method:
        cnn = HTTPConnection()
        cnn.connect(('www.google.com', 80))

        #create a GET request using the get method:
        request = cnn.get('/index.html')

        #finally perform the request to get a response:
        response = cnn.perform(request)

        #do something with the response:
        print response.body

    """

    log = logging.getLogger('HTTPConnection')

    def __init__(self):
        self.limit = None

    def connect(self, endpoint):
        """Connect to the webserver at *endpoint*. *endpoint* is a tuple (<host>, <port>)."""
        self._host = None
        if type(endpoint) == type(()):
            try:
                self._host = endpoint[0]
            except Exception:
                pass
        self._stream = BufferedStream(Connector.connect(endpoint), read_buffer_size = 1024 * 8, write_buffer_size = 1024 * 4)

    def set_limit(self, limit):
        self.limit = limit

    def receive(self):
        """Receive the next :class:`HTTPResponse` from the connection."""
        try:
            return self._receive()
        except TaskletExit:
            raise
        except EOFError:
            raise HTTPError("EOF while reading response")
        except HTTPError:
            raise
        except TimeoutError:
            raise
        except Exception:
            self.log.exception('')
            raise HTTPError("Exception while reading response")

    def _receive(self):

        response = HTTPResponse()

        with self._stream.get_reader() as reader:

            lines = reader.read_lines()

            #parse status line
            response.status = lines.next()

            #rest of response headers
            for line in lines:
                if not line: break
                key, value = line.split(': ')
                response.add_header(key, value)

            chunks = []

            if response.status_code != 204:
                #read data
                transfer_encoding = response.get_header('Transfer-Encoding', None)

                try:
                    content_length = int(response.get_header('Content-Length'))
                    if self.limit is not None and content_length > self.limit:
                        raise HTTPError("Response is too long")
                except Exception:
                    content_length = None

                #TODO better support large data, e.g. iterator instead of append all data to chunks

                if transfer_encoding == 'chunked':
                    while True:
                        chunk_line = reader.read_line()
                        chunk_size = int(chunk_line.split(';')[0], 16)
                        if chunk_size > 0:
                            data = reader.read_bytes(chunk_size)
                            reader.read_line() #chunk is always followed by a single empty line
                            chunks.append(data)
                        else:
                            reader.read_line() #chunk is always followed by a single empty line
                            break
                elif content_length is not None:
                    while content_length > 0:
                        n = min(CHUNK_SIZE, content_length)
                        data = reader.read_bytes(n)
                        chunks.append(data)
                        content_length -= len(data)
                else:
                    content_length = 0
                    while True:
                        try:
                            data = reader.read_bytes_available()
                        except EOFError:
                            break
                        chunks.append(data)
                        content_length += len(data)
                        if self.limit is not None and content_length > self.limit:
                            raise HTTPError("Response is too long")

            response.iter = chunks

            return response

    def get(self, path, host = None):
        """Returns a new :class:`HTTPRequest` with request.method = 'GET' and request.path = *path*.
        request.host will be set to the host used in :func:`connect`, or optionally you can specify a
        specific *host* just for this request.
        """
        request = HTTPRequest()
        request.method = 'GET'
        request.path = path
        request.host = host or self._host
        return request

    def post(self, path, body = None, host = None):
        """Returns a new :class:`HTTPRequest` with request.method = 'POST' and request.path = *path*.
        request.host will be set to the host used in :func:`connect`, or optionally you can specify a
        specific *host* just for this request.
        *body* is an optional string containing the data to post to the server.
        """
        request = HTTPRequest()
        request.method = 'POST'
        request.path = path
        request.host = host or self._host
        if body is not None:
            request.body = body
        return request

    def perform(self, request):
        """Sends the *request* and waits for and returns the :class:`HTTPResult`."""
        self.send(request)
        return self.receive()

    def send(self, request):
        """Sends the *request* on this connection."""
        if request.method is None:
            assert False, "request method must be set"
        if request.path is None:
            assert False, "request path must be set"
        if request.host is None:
            assert False, "request host must be set"

        with self._stream.get_writer() as writer:
            writer.clear()
            writer.write_bytes("%s %s HTTP/1.1\r\n" % (request.method, request.path))
            writer.write_bytes("Host: %s\r\n" % request.host)
            for header_name, header_value in request.headers:
                writer.write_bytes("%s: %s\r\n" % (header_name, header_value))
            writer.write_bytes("\r\n")
            if request.body is not None:
                writer.write_bytes(request.body)
            writer.flush()

    def close(self):
        """Close this connection."""
        self._stream.close()
Esempio n. 4
0
class HTTPConnection(object):
    """A HTTP 1.1 Client.

    Usage::

        #create an instance of this class and connect to a webserver using the connect method:
        cnn = HTTPConnection()
        cnn.connect(('www.google.com', 80))

        #create a GET request using the get method:
        request = cnn.get('/index.html')

        #finally perform the request to get a response:
        response = cnn.perform(request)

        #do something with the response:
        print response.body

    """

    log = logging.getLogger('HTTPConnection')

    def connect(self, endpoint):
        """Connect to the webserver at *endpoint*. *endpoint* is a tuple (<host>, <port>)."""
        self._host = None
        if type(endpoint) == type(()):
            try:
                self._host = endpoint[0]
            except:
                pass
        self._stream = BufferedStream(Connector.connect(endpoint),
                                      read_buffer_size=1024 * 8,
                                      write_buffer_size=1024 * 4)

    def receive(self):
        """Receive the next :class:`HTTPResponse` from the connection."""
        try:
            return self._receive()
        except TaskletExit:
            raise
        except EOFError:
            raise HTTPError("EOF while reading response")
        except Exception:
            self.log.exception('')
            raise HTTPError("Exception while reading response")

    def _receive(self):

        response = HTTPResponse()

        with self._stream.get_reader() as reader:

            lines = reader.read_lines()

            #parse status line
            response.status = lines.next()

            #rest of response headers
            for line in lines:
                if not line: break
                key, value = line.split(': ')
                response.add_header(key, value)

            #read data
            transfer_encoding = response.get_header('Transfer-Encoding', None)

            try:
                content_length = int(response.get_header('Content-Length'))
            except:
                content_length = None

            #TODO better support large data, e.g. iterator instead of append all data to chunks
            chunks = []

            if transfer_encoding == 'chunked':
                while True:
                    chunk_line = reader.read_line()
                    chunk_size = int(chunk_line.split(';')[0], 16)
                    if chunk_size > 0:
                        data = reader.read_bytes(chunk_size)
                        reader.read_line(
                        )  #chunk is always followed by a single empty line
                        chunks.append(data)
                    else:
                        reader.read_line(
                        )  #chunk is always followed by a single empty line
                        break
            elif content_length is not None:
                while content_length > 0:
                    n = min(CHUNK_SIZE, content_length)
                    data = reader.read_bytes(n)
                    chunks.append(data)
                    content_length -= len(data)
            else:
                assert False, 'TODO'

            response.iter = chunks

            return response

    def get(self, path, host=None):
        """Returns a new :class:`HTTPRequest` with request.method = 'GET' and request.path = *path*.
        request.host will be set to the host used in :func:`connect`, or optionally you can specify a
        specific *host* just for this request.
        """
        request = HTTPRequest()
        request.method = 'GET'
        request.path = path
        request.host = host or self._host
        return request

    def post(self, path, body=None, host=None):
        """Returns a new :class:`HTTPRequest` with request.method = 'POST' and request.path = *path*.
        request.host will be set to the host used in :func:`connect`, or optionally you can specify a
        specific *host* just for this request.
        *body* is an optional string containing the data to post to the server.
        """
        request = HTTPRequest()
        request.method = 'POST'
        request.path = path
        request.host = host or self._host
        if body is not None:
            request.body = body
        return request

    def perform(self, request):
        """Sends the *request* and waits for and returns the :class:`HTTPResult`."""
        self.send(request)
        return self.receive()

    def send(self, request):
        """Sends the *request* on this connection."""
        if request.method is None:
            assert False, "request method must be set"
        if request.path is None:
            assert False, "request path must be set"
        if request.host is None:
            assert False, "request host must be set"

        with self._stream.get_writer() as writer:
            writer.clear()
            writer.write_bytes("%s %s HTTP/1.1\r\n" %
                               (request.method, request.path))
            writer.write_bytes("Host: %s\r\n" % request.host)
            for header_name, header_value in request.headers:
                writer.write_bytes("%s: %s\r\n" % (header_name, header_value))
            writer.write_bytes("\r\n")
            if request.body is not None:
                writer.write_bytes(request.body)
            writer.flush()

    def close(self):
        """Close this connection."""
        self._stream.close()
Esempio n. 5
0
class MemcacheConnection(object):
    log = logging.getLogger("MemcacheConnection")

    _tasklet_pool = TaskletPool(worker_timeout = 2.0, 
                                worker_timeout_relative = False)

    def __init__(self, address, protocol = "text", codec = "default"):

        self._address = address

        self._stream = None
        self._read_queue = DeferredQueue(self._tasklet_pool.defer)
        self._write_queue = DeferredQueue(self._tasklet_pool.defer)

        self._protocol = MemcacheProtocol.create(protocol)
        self._protocol.set_codec(MemcacheCodec.create(codec))

    def connect(self):
        self._stream = BufferedStream(Socket.connect(self._address))

    def is_connected(self):
        return self._stream is not None

    def _defer_commands(self, cmds, result_channel):
        def _read_results():
            protocol = self._protocol
            with self._stream.get_reader() as reader:
                for cmd, args, error_value in cmds:
                    try:
                        result = protocol.read(cmd, reader)
                        result_channel.send(result)
                    except TaskletExit:
                        raise
                    except:
                        self.log.exception("read error in defer_commands")
                        result_channel.send((MemcacheResult.ERROR, error_value))
        #end _read_commands
        def _write_commands():
            protocol = self._protocol
            try:
                if not self.is_connected():
                    self.connect()
            except TaskletExit:
                raise
            except:
                self.log.exception("connect error in defer_commands")
                for _, _, error_value in cmds:
                    result_channel.send((MemcacheResult.ERROR, error_value))
                return
            with self._stream.get_writer() as writer:
                for cmd, args, error_value in cmds:
                    try:
                        protocol.write(cmd, writer, args)
                    except TaskletExit:
                        raise
                    except:
                        self.log.exception("write error in defer_commands")
                        result_channel.send((MemcacheResult.ERROR, error_value))
                writer.flush()
            self._read_queue.defer(_read_results)
        #end _write_commands
        self._write_queue.defer(_write_commands)

    def _defer_command(self, cmd, args, result_channel, error_value = None):
        self._defer_commands([(cmd, args, error_value)], result_channel)

    def _do_command(self, cmd, args, error_value = None):
        result_channel = ResultChannel()
        self._defer_command(cmd, args, result_channel, error_value)
        try:
            return result_channel.receive()
        except TimeoutError:
            return MemcacheResult.TIMEOUT, error_value

    def close(self):
        if self.is_connected():
            self._stream.close()
            self._stream = None

    def __setitem__(self, key, data):
        self.set(key, data)

    def __getitem__(self, key):
        return self.get(key)

    def delete(self, key, expiration = 0):
        return self._do_command("delete", (key, expiration))[0]

    def set(self, key, data, expiration = 0, flags = 0):
        return self._do_command("set", (key, data, expiration, flags))[0]

    def add(self, key, data, expiration = 0, flags = 0):
        return self._do_command("add", (key, data, expiration, flags))[0]

    def replace(self, key, data, expiration = 0, flags = 0):
        return self._do_command("replace", (key, data, expiration, flags))[0]

    def append(self, key, data, expiration = 0, flags = 0):
        return self._do_command("append", (key, data, expiration, flags))[0]

    def prepend(self, key, data, expiration = 0, flags = 0):
        return self._do_command("prepend", (key, data, expiration, flags))[0]

    def cas(self, key, data, cas_unique, expiration = 0, flags = 0):
        return self._do_command("cas", (key, data, expiration, flags, cas_unique))[0]

    def incr(self, key, increment):
        return self._do_command("incr", (key, increment))

    def decr(self, key, increment):
        return self._do_command("decr", (key, increment))

    def get(self, key, default = None):
        _, values = self._do_command("get", ([key], ), {})
        return values.get(key, default)

    def getr(self, key, default = None):
        result, values = self._do_command("get", ([key], ), {})
        return result, values.get(key, default)

    def gets(self, key, default = None):
        result, values = self._do_command("gets", ([key], ), {})
        value, cas_unique = values.get(key, (default, None))
        return result, value, cas_unique

    def get_multi(self, keys):
        return self._do_command("get", (keys, ))

    def gets_multi(self, keys):
        return self._do_command("gets", (keys, ))

    def version(self):
        return self._do_command("version", ())

    def batch(self):
        return CommandBatch(self)