Beispiel #1
0
 def http_connect(self, connect_to):
     self.wfile.write(
                 'CONNECT %s:%s HTTP/1.1\r\n'%tuple(connect_to) +
                 '\r\n'
                 )
     self.wfile.flush()
     l = self.rfile.readline()
     if not l:
         raise PathocError("Proxy CONNECT failed")
     parsed = http.parse_response_line(l)
     if not parsed[1] == 200:
         raise PathocError("Proxy CONNECT failed: %s - %s"%(parsed[1], parsed[2]))
     http.read_headers(self.rfile)
Beispiel #2
0
def create_http_request(flowheader, reqbuf):
    sfp = StringIO(reqbuf)
    method, url, httpversion = http.parse_init_http(sfp.readline())
    host, port, path = parse_url(url)
    headers = http.read_headers(sfp)

    if not host:
        if not headers.get("host"):
            host = flowheader.dstip
        else:
            host = headers.get("host")[0]
            if ":" in host:
                host = string.rsplit(host, ":", maxsplit=1)[0]

    if port == None:
        port = flowheader.dport

    # TODO: passing None as the second arg will produce and error if "expect" is in the headers
    content = http.read_http_body_request(sfp, None, headers, httpversion, None)

    # content = http.read_http_body(sfp, headers, True, None)
    return flow.Request(
        None,
        httpversion,
        host,
        port,
        "http",
        method,
        path,
        headers,
        content,
        flowheader.ts_request_start,
        flowheader.ts_request_finish,
    )
Beispiel #3
0
    def handle_request(self):
        """
            Returns a (again, log) tuple.

            again: True if request handling should continue.
            log: A dictionary, or None
        """
        line = self.rfile.readline()
        if line == "\r\n" or line == "\n": # Possible leftover from previous message
            line = self.rfile.readline()
        if line == "":
            # Normal termination
            return False, None

        m = utils.MemBool()
        if m(http.parse_init_connect(line)):
            headers = http.read_headers(self.rfile)
            self.wfile.write(
                        'HTTP/1.1 200 Connection established\r\n' +
                        ('Proxy-agent: %s\r\n'%version.NAMEVERSION) +
                        '\r\n'
                        )
            self.wfile.flush()
            if not self.server.ssloptions.not_after_connect:
                try:
                    self.convert_to_ssl(
                        self.server.ssloptions.certfile,
                        self.server.ssloptions.keyfile,
                    )
                except tcp.NetLibError, v:
                    s = str(v)
                    self.info(s)
                    return False, dict(type = "error", msg = s)
            return True, None
Beispiel #4
0
 def read_request(self, client_conn):
     if self.config.transparent_proxy:
         host, port = self.config.transparent_proxy["resolver"].original_addr(self.connection)
         if not self.ssl_established and (port in self.config.transparent_proxy["sslports"]):
             scheme = "https"
             certfile = self.find_cert(host, port, None)
             try:
                 self.convert_to_ssl(certfile, self.config.certfile or self.config.cacert)
             except tcp.NetLibError, v:
                 raise ProxyError(400, str(v))
         else:
             scheme = "http"
         host = self.sni or host
         line = self.get_line(self.rfile)
         if line == "":
             return None
         r = http.parse_init_http(line)
         if not r:
             raise ProxyError(400, "Bad HTTP request line: %s"%repr(line))
         method, path, httpversion = r
         headers = http.read_headers(self.rfile)
         content = http.read_http_body_request(
                     self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
                 )
         return flow.Request(client_conn, httpversion, host, port, scheme, method, path, headers, content)
Beispiel #5
0
 def http_connect(self, connect_to, wfile, rfile):
     wfile.write(
                 'CONNECT %s:%s HTTP/1.1\r\n'%tuple(connect_to) +
                 '\r\n'
                 )
     wfile.flush()
     rfile.readline()
     headers = http.read_headers(self.rfile)
Beispiel #6
0
    def from_stream(cls, rfile, include_content=True, body_size_limit=None):
        """
        Parse an HTTP request from a file stream
        """
        httpversion, host, port, scheme, method, path, headers, content, timestamp_start, timestamp_end \
            = None, None, None, None, None, None, None, None, None, None

        if hasattr(rfile, "reset_timestamps"):
            rfile.reset_timestamps()

        request_line = get_line(rfile)

        if hasattr(rfile, "first_byte_timestamp"):
            timestamp_start = rfile.first_byte_timestamp
        else:
            timestamp_start = utils.timestamp()

        request_line_parts = http.parse_init(request_line)
        if not request_line_parts:
            raise http.HttpError(
                400, "Bad HTTP request line: %s" % repr(request_line))
        method, path, httpversion = request_line_parts

        if path == '*':
            form_in = "asterisk"
        elif path.startswith("/"):
            form_in = "origin"
            if not netlib.utils.isascii(path):
                raise http.HttpError(
                    400, "Bad HTTP request line: %s" % repr(request_line))
        elif method.upper() == 'CONNECT':
            form_in = "authority"
            r = http.parse_init_connect(request_line)
            if not r:
                raise http.HttpError(
                    400, "Bad HTTP request line: %s" % repr(request_line))
            host, port, _ = r
            path = None
        else:
            form_in = "absolute"
            r = http.parse_init_proxy(request_line)
            if not r:
                raise http.HttpError(
                    400, "Bad HTTP request line: %s" % repr(request_line))
            _, scheme, host, port, path, _ = r

        headers = http.read_headers(rfile)
        if headers is None:
            raise http.HttpError(400, "Invalid headers")

        if include_content:
            content = http.read_http_body(rfile, headers, body_size_limit,
                                          True)
            timestamp_end = utils.timestamp()

        return HTTPRequest(form_in, method, scheme, host, port, path,
                           httpversion, headers, content, timestamp_start,
                           timestamp_end)
Beispiel #7
0
    def handle_request(self):
        """
            Returns True if handling should continue.
        """
        line = self.rfile.readline()
        if line == "\r\n" or line == "\n": # Possible leftover from previous message
            line = self.rfile.readline()
        if line == "":
            return

        parts = http.parse_init_http(line)
        if not parts:
            s = "Invalid first line: %s"%repr(line)
            self.info(s)
            self.server.add_log(
                dict(
                    type = "error",
                    msg = s
                )
            )
            return

        method, path, httpversion = parts
        headers = http.read_headers(self.rfile)
        if headers is None:
            s = "Invalid headers"
            self.info(s)
            self.server.add_log(
                dict(
                    type = "error",
                    msg = s
                )
            )
            return

        request_log = dict(
            path = path,
            method = method,
            headers = headers.lst,
            httpversion = httpversion,
            sni = self.sni,
            remote_address = self.client_address,
        )

        try:
            content = http.read_http_body_request(
                        self.rfile, self.wfile, headers, httpversion, None
                    )
        except http.HttpError, s:
            s = str(s)
            self.info(s)
            self.server.add_log(
                dict(
                    type = "error",
                    msg = s
                )
            )
            return
Beispiel #8
0
 def test_read_multi(self):
     data = """
         Header: one
         Header: two
         \r\n
     """
     data = textwrap.dedent(data)
     data = data.strip()
     s = cStringIO.StringIO(data)
     h = http.read_headers(s)
     assert h.lst == [["Header", "one"], ["Header", "two"]]
Beispiel #9
0
 def test_read_continued(self):
     data = """
         Header: one
         \ttwo
         Header2: three
         \r\n
     """
     data = textwrap.dedent(data)
     data = data.strip()
     s = cStringIO.StringIO(data)
     h = http.read_headers(s)
     assert h.lst == [["Header", "one\r\n two"], ["Header2", "three"]]
Beispiel #10
0
 def read_headers(self, authenticate=False):
     headers = http.read_headers(self.rfile)
     if headers is None:
         raise ProxyError(400, "Invalid headers")
     if authenticate and self.config.authenticator:
         if self.config.authenticator.authenticate(headers):
             self.config.authenticator.clean(headers)
         else:
             raise ProxyError(
                 407, "Proxy Authentication Required",
                 self.config.authenticator.auth_challenge_headers())
     return headers
Beispiel #11
0
    def from_stream(cls, rfile, include_content=True, body_size_limit=None):
        """
        Parse an HTTP request from a file stream
        """
        httpversion, host, port, scheme, method, path, headers, content, timestamp_start, timestamp_end \
            = None, None, None, None, None, None, None, None, None, None

        if hasattr(rfile, "reset_timestamps"):
            rfile.reset_timestamps()

        request_line = get_line(rfile)

        if hasattr(rfile, "first_byte_timestamp"):
            timestamp_start = rfile.first_byte_timestamp
        else:
            timestamp_start = utils.timestamp()

        request_line_parts = http.parse_init(request_line)
        if not request_line_parts:
            raise http.HttpError(400, "Bad HTTP request line: %s" % repr(request_line))
        method, path, httpversion = request_line_parts

        if path == '*':
            form_in = "asterisk"
        elif path.startswith("/"):
            form_in = "origin"
            if not netlib.utils.isascii(path):
                raise http.HttpError(400, "Bad HTTP request line: %s" % repr(request_line))
        elif method.upper() == 'CONNECT':
            form_in = "authority"
            r = http.parse_init_connect(request_line)
            if not r:
                raise http.HttpError(400, "Bad HTTP request line: %s" % repr(request_line))
            host, port, _ = r
            path = None
        else:
            form_in = "absolute"
            r = http.parse_init_proxy(request_line)
            if not r:
                raise http.HttpError(400, "Bad HTTP request line: %s" % repr(request_line))
            _, scheme, host, port, path, _ = r

        headers = http.read_headers(rfile)
        if headers is None:
            raise http.HttpError(400, "Invalid headers")

        if include_content:
            content = http.read_http_body(rfile, headers, body_size_limit, True)
            timestamp_end = utils.timestamp()

        return HTTPRequest(form_in, method, scheme, host, port, path, httpversion, headers, content,
                           timestamp_start, timestamp_end)
Beispiel #12
0
 def read_headers(self, authenticate=False):
     headers = http.read_headers(self.rfile)
     if headers is None:
         raise ProxyError(400, "Invalid headers")
     if authenticate and self.config.authenticator:
         if self.config.authenticator.authenticate(headers):
             self.config.authenticator.clean(headers)
         else:
             raise ProxyError(
                         407,
                         "Proxy Authentication Required",
                         self.config.authenticator.auth_challenge_headers()
                    )
     return headers
Beispiel #13
0
    def handle_request(self):
        """
            Returns a (again, log) tuple.

            again: True if request handling should continue.
            log: A dictionary, or None
        """
        if self.server.logreq:
            self.rfile.start_log()
        if self.server.logresp:
            self.wfile.start_log()

        line = http.get_request_line(self.rfile)
        if not line:
            # Normal termination
            return False

        m = utils.MemBool()
        if m(http.parse_init_connect(line)):
            headers = http.read_headers(self.rfile)
            self.wfile.write(
                'HTTP/1.1 200 Connection established\r\n' +
                ('Proxy-agent: %s\r\n' % version.NAMEVERSION) +
                '\r\n'
            )
            self.wfile.flush()
            if not self.server.ssloptions.not_after_connect:
                try:
                    cert, key, chain_file = self.server.ssloptions.get_cert(
                        m.v[0]
                    )
                    self.convert_to_ssl(
                        cert, key,
                        handle_sni=self.handle_sni,
                        request_client_cert=self.server.ssloptions.request_client_cert,
                        cipher_list=self.server.ssloptions.ciphers,
                        method=self.server.ssloptions.sslversion,
                    )
                except tcp.NetLibError, v:
                    s = str(v)
                    self.info(s)
                    self.addlog(dict(type="error", msg=s))
                    return False
            return True
Beispiel #14
0
    def handle_request(self):
        """
            Returns a (again, log) tuple. 

            again: True if request handling should continue.
            log: A dictionary, or None
        """
        line = self.rfile.readline()
        if line == "\r\n" or line == "\n": # Possible leftover from previous message
            line = self.rfile.readline()
        if line == "":
            # Normal termination
            return False, None

        parts = http.parse_init_http(line)
        if not parts:
            s = "Invalid first line: %s"%repr(line)
            self.info(s)
            return False, dict(type = "error", msg = s)

        method, path, httpversion = parts
        headers = http.read_headers(self.rfile)
        if headers is None:
            s = "Invalid headers"
            self.info(s)
            return False, dict(type = "error", msg = s)

        request_log = dict(
            path = path,
            method = method,
            headers = headers.lst,
            httpversion = httpversion,
            sni = self.sni,
            remote_address = self.client_address,
        )

        try:
            content = http.read_http_body_request(
                        self.rfile, self.wfile, headers, httpversion, None
                    )
        except http.HttpError, s:
            s = str(s)
            self.info(s)
            return False, dict(type = "error", msg = s)
Beispiel #15
0
    def handle_request(self):
        """
            Returns a (again, log) tuple. 

            again: True if request handling should continue.
            log: A dictionary, or None
        """
        line = self.rfile.readline()
        if line == "\r\n" or line == "\n":  # Possible leftover from previous message
            line = self.rfile.readline()
        if line == "":
            # Normal termination
            return False, None

        parts = http.parse_init_http(line)
        if not parts:
            s = "Invalid first line: %s" % repr(line)
            self.info(s)
            return False, dict(type="error", msg=s)

        method, path, httpversion = parts
        headers = http.read_headers(self.rfile)
        if headers is None:
            s = "Invalid headers"
            self.info(s)
            return False, dict(type="error", msg=s)

        request_log = dict(
            path=path,
            method=method,
            headers=headers.lst,
            httpversion=httpversion,
            sni=self.sni,
            remote_address=self.client_address,
        )

        try:
            content = http.read_http_body_request(self.rfile, self.wfile,
                                                  headers, httpversion, None)
        except http.HttpError, s:
            s = str(s)
            self.info(s)
            return False, dict(type="error", msg=s)
Beispiel #16
0
    def handle_request(self):
        """
            Returns a (again, log) tuple.

            again: True if request handling should continue.
            log: A dictionary, or None
        """
        if self.server.logreq:
            self.rfile.start_log()
        if self.server.logresp:
            self.wfile.start_log()

        line = http.get_request_line(self.rfile)
        if not line:
            # Normal termination
            return False

        m = utils.MemBool()
        if m(http.parse_init_connect(line)):
            headers = http.read_headers(self.rfile)
            self.wfile.write('HTTP/1.1 200 Connection established\r\n' +
                             ('Proxy-agent: %s\r\n' % version.NAMEVERSION) +
                             '\r\n')
            self.wfile.flush()
            if not self.server.ssloptions.not_after_connect:
                try:
                    cert, key, chain_file = self.server.ssloptions.get_cert(
                        m.v[0])
                    self.convert_to_ssl(
                        cert,
                        key,
                        handle_sni=self.handle_sni,
                        request_client_cert=self.server.ssloptions.
                        request_client_cert,
                        cipher_list=self.server.ssloptions.ciphers,
                        method=self.server.ssloptions.sslversion,
                    )
                except tcp.NetLibError, v:
                    s = str(v)
                    self.info(s)
                    self.addlog(dict(type="error", msg=s))
                    return False
            return True
Beispiel #17
0
 def read_request(self, client_conn):
     if self.config.transparent_proxy:
         host, port = self.config.transparent_proxy["resolver"].original_addr(self.connection)
         if not self.ssl_established and (port in self.config.transparent_proxy["sslports"]):
             scheme = "https"
             certfile = self.find_cert(host, port, None)
             self.convert_to_ssl(certfile, self.config.certfile or self.config.cacert)
         else:
             scheme = "http"
         host = self.sni or host
         line = self.get_line(self.rfile)
         if line == "":
             return None
         r = http.parse_init_http(line)
         if not r:
             raise ProxyError(400, "Bad HTTP request line: %s"%line)
         method, path, httpversion = r
         headers = http.read_headers(self.rfile)
         content = http.read_http_body_request(
                     self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
                 )
         return flow.Request(client_conn, httpversion, host, port, scheme, method, path, headers, content)
     elif self.config.reverse_proxy:
         line = self.get_line(self.rfile)
         if line == "":
             return None
         scheme, host, port = self.config.reverse_proxy
         r = http.parse_init_http(line)
         if not r:
             raise ProxyError(400, "Bad HTTP request line: %s"%line)
         method, path, httpversion = r
         headers = http.read_headers(self.rfile)
         content = http.read_http_body_request(
                     self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
                 )
         return flow.Request(client_conn, httpversion, host, port, "http", method, path, headers, content)
     else:
         line = self.get_line(self.rfile)
         if line == "":
             return None
         if line.startswith("CONNECT"):
             host, port, httpversion = http.parse_init_connect(line)
             # FIXME: Discard additional headers sent to the proxy. Should I expose
             # these to users?
             while 1:
                 d = self.rfile.readline()
                 if d == '\r\n' or d == '\n':
                     break
             self.wfile.write(
                         'HTTP/1.1 200 Connection established\r\n' +
                         ('Proxy-agent: %s\r\n'%version.NAMEVERSION) +
                         '\r\n'
                         )
             self.wfile.flush()
             certfile = self.find_cert(host, port, None)
             self.convert_to_ssl(certfile, self.config.certfile or self.config.cacert)
             self.proxy_connect_state = (host, port, httpversion)
             line = self.rfile.readline(line)
         if self.proxy_connect_state:
             host, port, httpversion = self.proxy_connect_state
             r = http.parse_init_http(line)
             if not r:
                 raise ProxyError(400, "Bad HTTP request line: %s"%line)
             method, path, httpversion = r
             headers = http.read_headers(self.rfile)
             content = http.read_http_body_request(
                 self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
             )
             return flow.Request(client_conn, httpversion, host, port, "https", method, path, headers, content)
         else:
             method, scheme, host, port, path, httpversion = http.parse_init_proxy(line)
             headers = http.read_headers(self.rfile)
             content = http.read_http_body_request(
                 self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
             )
             return flow.Request(client_conn, httpversion, host, port, scheme, method, path, headers, content)
Beispiel #18
0
     method, path, httpversion = r
     headers = http.read_headers(self.rfile)
     content = http.read_http_body_request(
                 self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
             )
     return flow.Request(client_conn, httpversion, host, port, scheme, method, path, headers, content)
 elif self.config.reverse_proxy:
     line = self.get_line(self.rfile)
     if line == "":
         return None
     scheme, host, port = self.config.reverse_proxy
     r = http.parse_init_http(line)
     if not r:
         raise ProxyError(400, "Bad HTTP request line: %s"%repr(line))
     method, path, httpversion = r
     headers = http.read_headers(self.rfile)
     content = http.read_http_body_request(
                 self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
             )
     return flow.Request(client_conn, httpversion, host, port, "http", method, path, headers, content)
 else:
     line = self.get_line(self.rfile)
     if line == "":
         return None
     if line.startswith("CONNECT"):
         r = http.parse_init_connect(line)
         if not r:
             raise ProxyError(400, "Bad HTTP request line: %s"%repr(line))
         host, port, httpversion = r
         # FIXME: Discard additional headers sent to the proxy. Should I expose
         # these to users?
Beispiel #19
0
 def _read(self, data, verbatim=False):
     if not verbatim:
         data = textwrap.dedent(data)
         data = data.strip()
     s = cStringIO.StringIO(data)
     return http.read_headers(s)
Beispiel #20
0
                    )
                except tcp.NetLibError, v:
                    s = str(v)
                    self.info(s)
                    return False, dict(type="error", msg=s)
            return True, None
        elif m(http.parse_init_proxy(line)):
            method, _, _, _, path, httpversion = m.v
        elif m(http.parse_init_http(line)):
            method, path, httpversion = m.v
        else:
            s = "Invalid first line: %s" % repr(line)
            self.info(s)
            return False, dict(type="error", msg=s)

        headers = http.read_headers(self.rfile)
        if headers is None:
            s = "Invalid headers"
            self.info(s)
            return False, dict(type="error", msg=s)

        clientcert = None
        if self.clientcert:
            clientcert = dict(
                cn=self.clientcert.cn,
                subject=self.clientcert.subject,
                serial=self.clientcert.serial,
                notbefore=self.clientcert.notbefore.isoformat(),
                notafter=self.clientcert.notafter.isoformat(),
                keyinfo=self.clientcert.keyinfo,
            )
Beispiel #21
0
 def _read(self, data, verbatim=False):
     if not verbatim:
         data = textwrap.dedent(data)
         data = data.strip()
     s = cStringIO.StringIO(data)
     return http.read_headers(s)
Beispiel #22
0
    def handle_request(self):
        """
            Returns a (again, log) tuple.

            again: True if request handling should continue.
            log: A dictionary, or None
        """
        line = self.rfile.readline()
        if line == "\r\n" or line == "\n":  # Possible leftover from previous message
            line = self.rfile.readline()
        if line == "":
            # Normal termination
            return False, None

        m = utils.MemBool()
        if m(http.parse_init_connect(line)):
            headers = http.read_headers(self.rfile)
            self.wfile.write(
                'HTTP/1.1 200 Connection established\r\n' +
                ('Proxy-agent: %s\r\n' % version.NAMEVERSION) +
                '\r\n'
            )
            self.wfile.flush()
            if not self.server.ssloptions.not_after_connect:
                try:
                    cert, key, chain_file = self.server.ssloptions.get_cert(m.v[0])
                    self.convert_to_ssl(
                        cert, key,
                        handle_sni=self.handle_sni,
                        request_client_cert=self.server.ssloptions.request_client_cert,
                        cipher_list=self.server.ssloptions.ciphers,
                        method=self.server.ssloptions.sslversion,
                    )
                except tcp.NetLibError as v:
                    s = str(v)
                    self.info(s)
                    return False, dict(type="error", msg=s)
            return True, None
        elif m(http.parse_init_proxy(line)):
            method, _, _, _, path, httpversion = m.v
        elif m(http.parse_init_http(line)):
            method, path, httpversion = m.v
        else:
            s = "Invalid first line: %s" % repr(line)
            self.info(s)
            return False, dict(type="error", msg=s)

        headers = http.read_headers(self.rfile)
        if headers is None:
            s = "Invalid headers"
            self.info(s)
            return False, dict(type="error", msg=s)

        clientcert = None
        if self.clientcert:
            clientcert = dict(
                cn=self.clientcert.cn,
                subject=self.clientcert.subject,
                serial=self.clientcert.serial,
                notbefore=self.clientcert.notbefore.isoformat(),
                notafter=self.clientcert.notafter.isoformat(),
                keyinfo=self.clientcert.keyinfo,
            )

        retlog = dict(
            type="crafted",
            request=dict(
                path=path,
                method=method,
                headers=headers.lst,
                httpversion=httpversion,
                sni=self.sni,
                remote_address=self.address(),
                clientcert=clientcert,
            ),
            cipher=None,
        )
        if self.ssl_established:
            retlog["cipher"] = self.get_current_cipher()

        try:
            content = http.read_http_body(
                self.rfile, headers, None,
                method, None, True
            )
        except http.HttpError as s:
            s = str(s)
            self.info(s)
            return False, dict(type="error", msg=s)

        for i in self.server.anchors:
            if i[0].match(path):
                self.info("crafting anchor: %s" % path)
                again, retlog["response"] = self.serve_crafted(i[1])
                return again, retlog

        if not self.server.nocraft and path.startswith(self.server.craftanchor):
            spec = urllib.parse.unquote(path)[len(self.server.craftanchor):]
            self.info("crafting spec: %s" % spec)
            try:
                crafted = language.parse_response(spec)
            except language.ParseException as v:
                self.info("Parse error: %s" % v.msg)
                crafted = language.make_error_response(
                    "Parse Error",
                    "Error parsing response spec: %s\n" % v.msg + v.marked()
                )
            again, retlog["response"] = self.serve_crafted(crafted)
            return again, retlog
        elif self.server.noweb:
            crafted = language.make_error_response("Access Denied")
            language.serve(crafted, self.wfile, self.server.request_settings)
            return False, dict(
                type="error",
                msg="Access denied: web interface disabled"
            )
        else:
            self.info("app: %s %s" % (method, path))
            req = wsgi.Request("http", method, path, headers, content)
            flow = wsgi.Flow(self.address, req)
            sn = self.connection.getsockname()
            a = wsgi.WSGIAdaptor(
                self.server.app,
                sn[0],
                self.server.address.port,
                version.NAMEVERSION
            )
            a.serve(flow, self.wfile)
            return True, None