Ejemplo n.º 1
0
    def __init__(self, mode="request"):
        super(HTTPParser, self).__init__()
        self.mode = mode

        self.charqueue = CharacterFIFO()
        self.lines = []
        self.readbuffer = ""
Ejemplo n.º 2
0
    def __init__(self, mode="request"):
        super(HTTPParser, self).__init__()
        self.mode = mode

        self.charqueue = CharacterFIFO()
        self.lines = []
        self.readbuffer = ""
Ejemplo n.º 3
0
 def __init__(self, chunksize=1048576, nodelay=False):
     super(Chunkifier, self).__init__()
     self.forwardqueue = CharacterFIFO()
     self.chunksize = chunksize
     self.nodelay = nodelay
Ejemplo n.º 4
0
class Chunkifier(component):
    """\
    Chunkifier([chunksize]) -> new Chunkifier component.
    
    Flow controller - collects incoming data and outputs it only as quanta of
    a given length in bytes (chunksize), unless the input stream ends (producerFinished). 
    
    Keyword arguments:
    - chunksize  -- Chunk size in bytes
    - nodelay    -- if set to True, partial chunks will be output rather
                    than buffering up data while waiting for more to arrive.
    """

    Inboxes = {
        "inbox": "Data stream to be split into chunks",
        "control": "Shut me down"
    }
    Outboxes = {
        "outbox": "Each message is a chunk",
        "signal": "I've shut down"
    }

    def __init__(self, chunksize=1048576, nodelay=False):
        super(Chunkifier, self).__init__()
        self.forwardqueue = CharacterFIFO()
        self.chunksize = chunksize
        self.nodelay = nodelay

    def sendPartialChunk(self):
        if len(self.forwardqueue) > 0:
            self.send(self.forwardqueue.poplength(len(self.forwardqueue)),
                      "outbox")

    def sendChunk(self):
        self.send(self.forwardqueue.poplength(self.chunksize), "outbox")

    def main(self):
        while 1:
            yield 1
            while self.dataReady("inbox"):
                msg = self.recv("inbox")

                # add this string to our character queue
                self.forwardqueue.push(msg)

            # while there is enough data to send another chunk
            while len(self.forwardqueue) >= self.chunksize:
                self.sendChunk()

            # if nodelay then send any data not yet sent rather than waiting for more
            if self.nodelay:
                self.sendPartialChunk()

            while self.dataReady("control"):
                msg = self.recv("control")
                if isinstance(msg, producerFinished):
                    self.sendPartialChunk()
                    self.send(producerFinished(self), "signal")
                    return
                elif isinstance(msg, shutdown):
                    self.send(producerFinished(self), "signal")
                    return
            self.pause()
Ejemplo n.º 5
0
class HTTPParser(component):
    """Component that transforms HTTP requests or responses from a
    single TCP connection into multiple easy-to-use dictionary objects."""

    # if we don't set a maximum URI length, then someone could just send us a few GB of URI
    # which we'd try to store in memory - not nice!
    MaxFirstLineLength = 10000
    MaxTotalHeadersLength = 50000

    Inboxes = {"inbox": "Raw HTTP requests/responses", "control": "UNUSED"}
    Outboxes = {
        "outbox": "HTTP request object",
        "debug": "Debugging information",
        "signal": "UNUSED"
    }

    def __init__(self, mode="request"):
        super(HTTPParser, self).__init__()
        self.mode = mode

        self.charqueue = CharacterFIFO()
        self.lines = []
        self.readbuffer = ""

    def splitProtocolVersion(self, protvers, requestobject):
        protvers = protvers.split("/")
        if len(protvers) != 2:
            requestobject["protocol"] = protvers[0]
            requestobject["version"] = "0.0"
        else:
            requestobject["protocol"] = protvers[0]
            requestobject["version"] = protvers[1]

    def dataFetch(self):
        """Read once from inbox (generally a TCP connection) and add
        what is received to the readbuffer. This is somewhat inefficient for long lines maybe O(n^2)"""
        if self.dataReady("inbox"):
            self.charqueue.push(self.recv("inbox"))
            return 1
        else:
            return 0

    def debug(self, msg):
        # self.send(msg, "debug")
        print msg

    def shouldShutdown(self):
        while self.dataReady("control"):
            temp = self.recv("control")
            if isinstance(temp, shutdown):
                self.debug("HTTPParser should shutdown")
                return True

        return False

    def bufferLength(self):
        return len(self.charqueue)

    def popBuffer(self):
        return self.charqueue.poplength(len(self.charqueue))

    def nextLine(self):
        "Fetch the next complete line in the readbuffer, if there is one"

        line = self.charqueue.popline()
        if line != None:
            if line[-2:] == "\r\n":
                line = line[:-2]
            elif line[-1:] == "\n":
                line = line[:-1]
            self.debug("Fetched line: " + line)
        return line

    def main(self):

        while 1:
            self.debug("HTTPParser::main - stage 0")

            if self.mode == "request":
                requestobject = {
                    "bad": False,
                    "headers": {},
                    "raw-uri": "",
                    "version": "0.9",
                    "method": "",
                    "protocol": "",
                    "body": ""
                }
            else:
                requestobject = {
                    "bad": False,
                    "headers": {},
                    "responsecode": "",
                    "version": "0.9",
                    "method": "",
                    "protocol": "",
                    "body": ""
                }

            self.debug("HTTPParser::main - awaiting initial line")
            #state 1 - awaiting initial line
            currentline = None
            while currentline == None:
                self.debug("HTTPParser::main - stage 1")
                if self.shouldShutdown(): return
                while self.dataFetch():
                    pass
                currentline = self.nextLine()
                if currentline == None:
                    if self.bufferLength(
                    ) > self.MaxFirstLineLength:  # looks like a DoS attempt - reject line rather than wasting lots of memory
                        self.debug(
                            "HTTPParser request line too long - terminating\n")
                        self.send(
                            producerFinished(self),
                            "signal")  #this functionality is semi-complete
                        return
                    else:
                        self.pause()
                        yield 1

            self.debug("HTTPParser::main - initial line found")
            splitline = string.split(currentline, " ")

            if self.mode == "request":
                # e.g. GET / HTTP/1.0
                if len(splitline) < 2:
                    requestobject["bad"] = True
                elif len(splitline) == 2:
                    # must be HTTP/0.9
                    requestobject["method"] = splitline[0]
                    requestobject["raw-uri"] = splitline[1]
                    requestobject["protocol"] = "HTTP"
                    requestobject["version"] = "0.9"
                else:  #deal with normal HTTP including badly formed URIs
                    requestobject["method"] = splitline[0]

                    #next line supports all working clients but also
                    #some broken clients that don't encode spaces properly!
                    #update is used to merge the uri-dictionary with the request object
                    requestobject.update(
                        splitUri(string.join(splitline[1:-1], "%20")))
                    self.splitProtocolVersion(splitline[-1], requestobject)

                    #if requestobject["protocol"] != "HTTP":
                    #    requestobject["bad"] = True
            else:
                #e.g. HTTP/1.1 200 OK that's fine
                if len(splitline) < 2:
                    requestobject["bad"] = True
                else:
                    requestobject["responsecode"] = splitline[1]
                    self.splitProtocolVersion(splitline[0], requestobject)

            requestobject["version"] = TuplifyHTTPVersion(
                requestobject["version"])

            if not requestobject["bad"]:
                # some requests, e.g. POSTs, come with a payload that we must accept after the headers
                # others, e.g. GETs, have no payload - after the headers comes the next request
                # responses (i.e. from a server) always have a payload
                if self.mode == "response" or requestobject["method"] in [
                        "PUT", "POST"
                ]:
                    bodiedrequest = True
                else:
                    bodiedrequest = False

                # state 2 - as this is a valid request, we now accept headers
                # TODO: add a limit to header size
                previousheader = ""
                endofheaders = False
                while not endofheaders:
                    self.debug("HTTPParser::main - stage 2")
                    if self.shouldShutdown(): return
                    while self.dataFetch():
                        pass

                    currentline = self.nextLine()
                    while currentline != None:
                        self.debug("HTTPParser::main - stage 2.1")
                        if currentline == "":
                            #print "End of headers found"
                            endofheaders = True
                            break
                        else:
                            if currentline[0] in [" ",
                                                  "\t"]:  #continued header
                                requestobject["headers"][
                                    previousheader] += " " + string.lstrip(
                                        currentline)
                            else:
                                splitheader = string.split(currentline, ":")
                                #print "Found header: " + splitheader[0]
                                requestobject["headers"][string.lower(
                                    splitheader[0])] = string.lstrip(
                                        currentline[len(splitheader[0]) + 1:])
                        currentline = self.nextLine()
                        #should parse headers header
                    if not endofheaders:
                        self.pause()
                        yield 1

                self.debug("HTTPParser::main - stage 2 complete")
                if requestobject["headers"].has_key("host"):
                    requestobject["uri-server"] = requestobject["headers"][
                        "host"]

                if requestobject["version"] > (1, 1):
                    requestobject["headers"]["connection"] = requestobject[
                        "headers"].get("connection", "keep-alive")
                else:
                    requestobject["headers"]["connection"] = requestobject[
                        "headers"].get("connection", "close")

                # The header section is complete, so send it on.
                self.send(ParsedHTTPHeader(requestobject), "outbox")
                if bodiedrequest:
                    self.debug("HTTPParser::main - stage 3 start")
                    #state 3 - the headers are complete - awaiting the message
                    if requestobject["headers"].get("transfer-encoding",
                                                    "").lower() == "chunked":
                        bodylength = -1
                        while bodylength != 0:
                            self.debug("HTTPParser::main - stage 3.chunked")
                            currentline = None
                            while currentline == None:
                                self.debug(
                                    "HTTPParser::main - stage 3.chunked.1")
                                if self.shouldShutdown(): return
                                while self.dataFetch():
                                    pass
                                currentline = self.nextLine()
                                if currentline == None:
                                    self.pause()
                                    yield 1
                            #print requestobject
                            splitline = currentline.split(";")

                            try:
                                bodylength = string.atoi(splitline[0], 16)
                            except ValueError:
                                print "Warning: bad chunk length in request/response being parsed by HTTPParser"
                                bodylength = 0
                                requestobject["bad"] = True

                            self.debug(
                                "HTTPParser::main - chunking: '%s' '%s' %d" %
                                (currentline, splitline, bodylength))

                            if bodylength != 0:
                                while len(self.readbuffer) < bodylength:
                                    self.debug(
                                        "HTTPParser::main - stage 3.chunked.2")
                                    if self.shouldShutdown(): return
                                    while self.dataFetch():
                                        pass

                                    if len(self.readbuffer) < bodylength:
                                        self.pause()
                                        yield 1
                                # we could do better than this - this will eat memory when overly large chunks are used
                                self.send(
                                    ParsedHTTPBodyChunk(
                                        self.readbuffer[:bodylength]),
                                    "outbox")

                            if self.readbuffer[bodylength:bodylength +
                                               2] == "\r\n":
                                self.readbuffer = self.readbuffer[bodylength +
                                                                  2:]
                            elif self.readbuffer[bodylength:bodylength +
                                                 1] == "\n":
                                self.readbuffer = self.readbuffer[bodylength +
                                                                  1:]
                            else:
                                print "Warning: no trailing new line on chunk in HTTPParser"
                                requestobject["bad"] = True
                                break

                            if bodylength == 0:
                                break
                    elif requestobject["headers"].has_key("content-length"):
                        if string.lower(requestobject["headers"].get(
                                "expect", "")) == "100-continue":
                            # we're supposed to say continue, but this is a pain
                            # and everything still works if we don't just with a few secs delay
                            pass
                        self.debug(
                            "HTTPParser::main - stage 3.length-known start")

                        bodylengthremaining = int(
                            requestobject["headers"]["content-length"])

                        while bodylengthremaining > 0:
                            #print "HTTPParser::main - stage 3.length known.1"
                            if self.shouldShutdown(): return
                            while self.dataFetch():
                                pass

                            if bodylengthremaining < len(
                                    self.readbuffer
                            ):  #i.e. we have some extra data from the next request
                                self.send(
                                    ParsedHTTPBodyChunk(
                                        self.readbuffer[:bodylengthremaining]),
                                    "outbox")
                                self.readbuffer = self.readbuffer[
                                    bodylengthremaining:]
                                bodylengthremaining = 0
                            elif len(self.readbuffer) > 0:
                                bodylengthremaining -= len(self.readbuffer)
                                self.send(ParsedHTTPBodyChunk(self.readbuffer),
                                          "outbox")
                                self.readbuffer = ""

                            if bodylengthremaining > 0:
                                self.pause()
                                yield 1

                        self.readbuffer = self.readbuffer[
                            bodylengthremaining:]  #for the next request
                    else:  # we'll assume it's a connection: close jobby
                        # THIS CODE IS BROKEN - IT WILL NOT TERMINATE UNTIL CSA SIGNALS HALF-CLOSURE OF CONNECTIONS!
                        self.debug(
                            "HTTPParser::main - stage 3.connection-close start\n"
                        )
                        connectionopen = True
                        while connectionopen:
                            self.debug(
                                "HTTPParser::main - stage 3.connection close.1"
                            )
                            if self.shouldShutdown(): return

                            while self.dataFetch():
                                self.send(
                                    ParsedHTTPBodyChunk(self.popBuffer()),
                                    "outbox")

                            while self.dataReady("control"):
                                #print "!"
                                temp = self.recv("control")
                                if isinstance(temp, producerFinished):
                                    connectionopen = False
                                    break
                                elif isinstance(temp, shutdown):
                                    return

                            if connectionopen:
                                self.pause()
                                yield 1
                    #else:
                    #    #no way of knowing how long the body is
                    #    requestobject["bad"] = 411 #length required
                    #    #print "HTTPParser::main - stage 3.bad"

                #state 4 - request complete, send it on
            self.debug("HTTPParser::main - request sent on\n")
            #print requestobject

            self.send(ParsedHTTPEnd(), "outbox")
            if string.lower(requestobject["headers"].get("connection",
                                                         "")) == "close":
                self.debug("HTTPParser connection close\n")
                self.send(producerFinished(self),
                          "signal")  #this functionality is semi-complete
                return
Ejemplo n.º 6
0
 def __init__(self, chunksize = 1048576, nodelay = False):
     super(Chunkifier, self).__init__()
     self.forwardqueue = CharacterFIFO()
     self.chunksize = chunksize
     self.nodelay = nodelay
Ejemplo n.º 7
0
class Chunkifier(component):
    """\
    Chunkifier([chunksize]) -> new Chunkifier component.
    
    Flow controller - collects incoming data and outputs it only as quanta of
    a given length in bytes (chunksize), unless the input stream ends (producerFinished). 
    
    Keyword arguments:
    - chunksize  -- Chunk size in bytes
    - nodelay    -- if set to True, partial chunks will be output rather
                    than buffering up data while waiting for more to arrive.
    """
    
    Inboxes = {
        "inbox" : "Data stream to be split into chunks",
        "control": "Shut me down"
    }
    Outboxes = {
        "outbox" : "Each message is a chunk",
        "signal": "I've shut down"
    }

    def __init__(self, chunksize = 1048576, nodelay = False):
        super(Chunkifier, self).__init__()
        self.forwardqueue = CharacterFIFO()
        self.chunksize = chunksize
        self.nodelay = nodelay
    
    def sendPartialChunk(self):
        if len(self.forwardqueue) > 0:
            self.send(self.forwardqueue.poplength(len(self.forwardqueue)), "outbox")
    
    def sendChunk(self):
        self.send(self.forwardqueue.poplength(self.chunksize), "outbox")

    def main(self):
        while 1:
            yield 1
            while self.dataReady("inbox"):
                msg = self.recv("inbox")
                
                # add this string to our character queue
                self.forwardqueue.push(msg)
            
            # while there is enough data to send another chunk
            while len(self.forwardqueue) >= self.chunksize:
                self.sendChunk()
            
            # if nodelay then send any data not yet sent rather than waiting for more
            if self.nodelay:
                self.sendPartialChunk()
                
            while self.dataReady("control"):
                msg = self.recv("control")
                if isinstance(msg, producerFinished):
                    self.sendPartialChunk()
                    self.send(producerFinished(self), "signal")
                    return
                elif isinstance(msg, shutdown):
                    self.send(producerFinished(self), "signal")
                    return
            self.pause()
Ejemplo n.º 8
0
class HTTPParser(component):
    """Component that transforms HTTP requests or responses from a
    single TCP connection into multiple easy-to-use dictionary objects."""
    
    # if we don't set a maximum URI length, then someone could just send us a few GB of URI
    # which we'd try to store in memory - not nice!
    MaxFirstLineLength = 10000
    MaxTotalHeadersLength = 50000
    
    Inboxes =  {
        "inbox"         : "Raw HTTP requests/responses",
        "control"       : "UNUSED"
    }
    Outboxes = {
        "outbox"        : "HTTP request object",
        "debug"         : "Debugging information",
        "signal"        : "UNUSED"
    }
    
    def __init__(self, mode="request"):
        super(HTTPParser, self).__init__()
        self.mode = mode

        self.charqueue = CharacterFIFO()
        self.lines = []
        self.readbuffer = ""
        
    def splitProtocolVersion(self, protvers, requestobject):
        protvers = protvers.split("/")
        if len(protvers) != 2:
            requestobject["protocol"] = protvers[0]
            requestobject["version"] = "0.0"
        else:
            requestobject["protocol"] = protvers[0]
            requestobject["version"]  = protvers[1]
    
    def dataFetch(self):
        """Read once from inbox (generally a TCP connection) and add
        what is received to the readbuffer. This is somewhat inefficient for long lines maybe O(n^2)"""
        if self.dataReady("inbox"):
            self.charqueue.push(self.recv("inbox"))
            return 1
        else:
            return 0
            
    def debug(self, msg):
        # self.send(msg, "debug")
        print msg
    
    def shouldShutdown(self):
        while self.dataReady("control"):
            temp = self.recv("control")
            if isinstance(temp, shutdown):
                self.debug("HTTPParser should shutdown")
                return True
        
        return False
    
    def bufferLength(self):
        return len(self.charqueue)
    
    def popBuffer(self):
        return self.charqueue.poplength(len(self.charqueue))
        
    def nextLine(self):
        "Fetch the next complete line in the readbuffer, if there is one"

        line  = self.charqueue.popline()
        if line != None:
            if line[-2:] == "\r\n":
                line = line[:-2]
            elif line[-1:] == "\n":
                line = line[:-1]
            self.debug("Fetched line: " + line)
        return line
    
    def main(self):

        while 1:
            self.debug("HTTPParser::main - stage 0")
            
            if self.mode == "request":
                requestobject = { "bad": False,
                                  "headers": {},
                                  "raw-uri": "",
                                  "version": "0.9",
                                  "method": "",
                                  "protocol": "",
                                  "body": "" }
            else:
                requestobject = { "bad": False,
                                  "headers": {},
                                  "responsecode": "",
                                  "version": "0.9",
                                  "method": "",
                                  "protocol": "",
                                  "body": "" }
           
            self.debug("HTTPParser::main - awaiting initial line")
            #state 1 - awaiting initial line
            currentline = None
            while currentline == None:
                self.debug("HTTPParser::main - stage 1")
                if self.shouldShutdown(): return
                while self.dataFetch():
                    pass
                currentline = self.nextLine()
                if currentline == None:
                    if self.bufferLength() > self.MaxFirstLineLength: # looks like a DoS attempt - reject line rather than wasting lots of memory
                        self.debug("HTTPParser request line too long - terminating\n")
                        self.send(producerFinished(self), "signal") #this functionality is semi-complete
                        return
                    else:
                        self.pause()
                        yield 1
                
            self.debug("HTTPParser::main - initial line found")
            splitline = string.split(currentline, " ")
            
            if self.mode == "request":
                # e.g. GET / HTTP/1.0
                if len(splitline) < 2:
                    requestobject["bad"] = True
                elif len(splitline) == 2:
                    # must be HTTP/0.9
                    requestobject["method"] = splitline[0]
                    requestobject["raw-uri"] = splitline[1]
                    requestobject["protocol"] = "HTTP"
                    requestobject["version"] = "0.9"
                else: #deal with normal HTTP including badly formed URIs
                    requestobject["method"] = splitline[0]

                    #next line supports all working clients but also 
                    #some broken clients that don't encode spaces properly!
                    #update is used to merge the uri-dictionary with the request object
                    requestobject.update(splitUri(string.join(splitline[1:-1], "%20")))
                    self.splitProtocolVersion(splitline[-1], requestobject)
                    
                    #if requestobject["protocol"] != "HTTP":
                    #    requestobject["bad"] = True
            else:
                #e.g. HTTP/1.1 200 OK that's fine
                if len(splitline) < 2:
                    requestobject["bad"] = True
                else:
                    requestobject["responsecode"] = splitline[1]
                    self.splitProtocolVersion(splitline[0], requestobject)
            
            requestobject["version"] = TuplifyHTTPVersion(requestobject["version"])
            
            if not requestobject["bad"]:
                # some requests, e.g. POSTs, come with a payload that we must accept after the headers
                # others, e.g. GETs, have no payload - after the headers comes the next request
                # responses (i.e. from a server) always have a payload
                if self.mode == "response" or requestobject["method"] in ["PUT", "POST"]:
                    bodiedrequest = True
                else:
                    bodiedrequest = False
                
                # state 2 - as this is a valid request, we now accept headers
                # TODO: add a limit to header size
                previousheader = ""
                endofheaders = False
                while not endofheaders:
                    self.debug("HTTPParser::main - stage 2")
                    if self.shouldShutdown(): return						
                    while self.dataFetch():
                        pass
                        
                    currentline = self.nextLine()
                    while currentline != None:
                        self.debug("HTTPParser::main - stage 2.1")
                        if currentline == "":
                            #print "End of headers found"
                            endofheaders = True
                            break
                        else:
                            if currentline[0] in [" ", "\t"]: #continued header
                                requestobject["headers"][previousheader] += " " + string.lstrip(currentline)
                            else:
                                splitheader = string.split(currentline, ":")
                                #print "Found header: " + splitheader[0]
                                requestobject["headers"][string.lower(splitheader[0])] = string.lstrip(currentline[len(splitheader[0]) + 1:])
                        currentline = self.nextLine()
                        #should parse headers header
                    if not endofheaders:
                        self.pause()
                        yield 1

                self.debug("HTTPParser::main - stage 2 complete")
                if requestobject["headers"].has_key("host"):
                    requestobject["uri-server"] = requestobject["headers"]["host"]
                
                if requestobject["version"] > (1, 1):
                    requestobject["headers"]["connection"] = requestobject["headers"].get("connection", "keep-alive")
                else:
                    requestobject["headers"]["connection"] = requestobject["headers"].get("connection", "close")

                # The header section is complete, so send it on.
                self.send(ParsedHTTPHeader(requestobject), "outbox")
                if bodiedrequest:
                    self.debug("HTTPParser::main - stage 3 start")
                    #state 3 - the headers are complete - awaiting the message
                    if requestobject["headers"].get("transfer-encoding","").lower() == "chunked":
                        bodylength = -1
                        while bodylength != 0:
                            self.debug("HTTPParser::main - stage 3.chunked")                        
                            currentline = None
                            while currentline == None:
                                self.debug("HTTPParser::main - stage 3.chunked.1")
                                if self.shouldShutdown(): return
                                while self.dataFetch():
                                    pass
                                currentline = self.nextLine()
                                if currentline == None:
                                    self.pause()
                                    yield 1
                            #print requestobject
                            splitline = currentline.split(";")

                            try:
                                bodylength = string.atoi(splitline[0], 16)
                            except ValueError:
                                print "Warning: bad chunk length in request/response being parsed by HTTPParser"
                                bodylength = 0
                                requestobject["bad"] = True

                            self.debug("HTTPParser::main - chunking: '%s' '%s' %d" % (currentline, splitline, bodylength))
                             
                            if bodylength != 0:                                
                                while len(self.readbuffer) < bodylength:
                                    self.debug("HTTPParser::main - stage 3.chunked.2")
                                    if self.shouldShutdown(): return						
                                    while self.dataFetch():
                                        pass
                                        
                                    if len(self.readbuffer) < bodylength:
                                        self.pause()
                                        yield 1
                                # we could do better than this - this will eat memory when overly large chunks are used
                                self.send(ParsedHTTPBodyChunk(self.readbuffer[:bodylength]), "outbox")
                            
                            if self.readbuffer[bodylength:bodylength + 2] == "\r\n":
                                self.readbuffer = self.readbuffer[bodylength + 2:]
                            elif self.readbuffer[bodylength:bodylength + 1] == "\n":
                                self.readbuffer = self.readbuffer[bodylength + 1:]
                            else:
                                print "Warning: no trailing new line on chunk in HTTPParser"
                                requestobject["bad"] = True
                                break
                            
                            if bodylength == 0:
                                break
                    elif requestobject["headers"].has_key("content-length"):
                        if string.lower(requestobject["headers"].get("expect", "")) == "100-continue":
                            # we're supposed to say continue, but this is a pain
                            # and everything still works if we don't just with a few secs delay
                            pass
                        self.debug("HTTPParser::main - stage 3.length-known start")
                        
                        bodylengthremaining = int(requestobject["headers"]["content-length"])
                         
                        while bodylengthremaining > 0:
                            #print "HTTPParser::main - stage 3.length known.1"
                            if self.shouldShutdown(): return						
                            while self.dataFetch():
                                pass
                             
                            if bodylengthremaining < len(self.readbuffer): #i.e. we have some extra data from the next request
                                self.send(ParsedHTTPBodyChunk(self.readbuffer[:bodylengthremaining]), "outbox")
                                self.readbuffer = self.readbuffer[bodylengthremaining:]
                                bodylengthremaining = 0
                            elif len(self.readbuffer) > 0:
                                bodylengthremaining -= len(self.readbuffer)
                                self.send(ParsedHTTPBodyChunk(self.readbuffer), "outbox")
                                self.readbuffer = ""
                                
                            if bodylengthremaining > 0:
                                self.pause()
                                yield 1
                        
                        self.readbuffer = self.readbuffer[bodylengthremaining:] #for the next request
                    else: # we'll assume it's a connection: close jobby
                        # THIS CODE IS BROKEN - IT WILL NOT TERMINATE UNTIL CSA SIGNALS HALF-CLOSURE OF CONNECTIONS!
                        self.debug("HTTPParser::main - stage 3.connection-close start\n")
                        connectionopen = True
                        while connectionopen:
                            self.debug("HTTPParser::main - stage 3.connection close.1")
                            if self.shouldShutdown(): return
                            
                            while self.dataFetch():
                                self.send(ParsedHTTPBodyChunk(self.popBuffer()), "outbox")
                                                                
                            while self.dataReady("control"):
                                #print "!"                            
                                temp = self.recv("control")
                                if isinstance(temp, producerFinished):
                                    connectionopen = False
                                    break
                                elif isinstance(temp, shutdown):
                                    return
                            
                            
                            if connectionopen:
                                self.pause()
                                yield 1
                    #else:
                    #    #no way of knowing how long the body is
                    #    requestobject["bad"] = 411 #length required
                    #    #print "HTTPParser::main - stage 3.bad"

                #state 4 - request complete, send it on
            self.debug("HTTPParser::main - request sent on\n")
            #print requestobject
            
                    
            self.send(ParsedHTTPEnd(), "outbox")
            if string.lower(requestobject["headers"].get("connection", "")) == "close":
                self.debug("HTTPParser connection close\n")
                self.send(producerFinished(self), "signal") #this functionality is semi-complete
                return