Beispiel #1
0
def test_prase_init_proxy():
    u = "GET http://foo.com:8888/test HTTP/1.1"
    m, s, h, po, pa, httpversion = http.parse_init_proxy(u)
    assert m == "GET"
    assert s == "http"
    assert h == "foo.com"
    assert po == 8888
    assert pa == "/test"
    assert httpversion == (1, 1)

    assert not http.parse_init_proxy("invalid")
    assert not http.parse_init_proxy("GET invalid HTTP/1.1")
    assert not http.parse_init_proxy("GET http://foo.com:8888/test foo/1.1")
Beispiel #2
0
def test_prase_init_proxy():
    u = "GET http://foo.com:8888/test HTTP/1.1"
    m, s, h, po, pa, httpversion = http.parse_init_proxy(u)
    assert m == "GET"
    assert s == "http"
    assert h == "foo.com"
    assert po == 8888
    assert pa == "/test"
    assert httpversion == (1, 1)

    assert not http.parse_init_proxy("invalid")
    assert not http.parse_init_proxy("GET invalid HTTP/1.1")
    assert not http.parse_init_proxy("GET http://foo.com:8888/test foo/1.1")
Beispiel #3
0
    def from_stream(cls, rfile, include_content=True, body_size_limit=None):
        """
        Parse an HTTP request from a file stream
        """
        httpversion, host, port, scheme, method, path, headers, content, timestamp_start, timestamp_end \
            = None, None, None, None, None, None, None, None, None, None

        if hasattr(rfile, "reset_timestamps"):
            rfile.reset_timestamps()

        request_line = get_line(rfile)

        if hasattr(rfile, "first_byte_timestamp"):
            timestamp_start = rfile.first_byte_timestamp
        else:
            timestamp_start = utils.timestamp()

        request_line_parts = http.parse_init(request_line)
        if not request_line_parts:
            raise http.HttpError(
                400, "Bad HTTP request line: %s" % repr(request_line))
        method, path, httpversion = request_line_parts

        if path == '*':
            form_in = "asterisk"
        elif path.startswith("/"):
            form_in = "origin"
            if not netlib.utils.isascii(path):
                raise http.HttpError(
                    400, "Bad HTTP request line: %s" % repr(request_line))
        elif method.upper() == 'CONNECT':
            form_in = "authority"
            r = http.parse_init_connect(request_line)
            if not r:
                raise http.HttpError(
                    400, "Bad HTTP request line: %s" % repr(request_line))
            host, port, _ = r
            path = None
        else:
            form_in = "absolute"
            r = http.parse_init_proxy(request_line)
            if not r:
                raise http.HttpError(
                    400, "Bad HTTP request line: %s" % repr(request_line))
            _, scheme, host, port, path, _ = r

        headers = http.read_headers(rfile)
        if headers is None:
            raise http.HttpError(400, "Invalid headers")

        if include_content:
            content = http.read_http_body(rfile, headers, body_size_limit,
                                          True)
            timestamp_end = utils.timestamp()

        return HTTPRequest(form_in, method, scheme, host, port, path,
                           httpversion, headers, content, timestamp_start,
                           timestamp_end)
Beispiel #4
0
    def from_stream(cls, rfile, include_content=True, body_size_limit=None):
        """
        Parse an HTTP request from a file stream
        """
        httpversion, host, port, scheme, method, path, headers, content, timestamp_start, timestamp_end \
            = None, None, None, None, None, None, None, None, None, None

        if hasattr(rfile, "reset_timestamps"):
            rfile.reset_timestamps()

        request_line = get_line(rfile)

        if hasattr(rfile, "first_byte_timestamp"):
            timestamp_start = rfile.first_byte_timestamp
        else:
            timestamp_start = utils.timestamp()

        request_line_parts = http.parse_init(request_line)
        if not request_line_parts:
            raise http.HttpError(400, "Bad HTTP request line: %s" % repr(request_line))
        method, path, httpversion = request_line_parts

        if path == '*':
            form_in = "asterisk"
        elif path.startswith("/"):
            form_in = "origin"
            if not netlib.utils.isascii(path):
                raise http.HttpError(400, "Bad HTTP request line: %s" % repr(request_line))
        elif method.upper() == 'CONNECT':
            form_in = "authority"
            r = http.parse_init_connect(request_line)
            if not r:
                raise http.HttpError(400, "Bad HTTP request line: %s" % repr(request_line))
            host, port, _ = r
            path = None
        else:
            form_in = "absolute"
            r = http.parse_init_proxy(request_line)
            if not r:
                raise http.HttpError(400, "Bad HTTP request line: %s" % repr(request_line))
            _, scheme, host, port, path, _ = r

        headers = http.read_headers(rfile)
        if headers is None:
            raise http.HttpError(400, "Invalid headers")

        if include_content:
            content = http.read_http_body(rfile, headers, body_size_limit, True)
            timestamp_end = utils.timestamp()

        return HTTPRequest(form_in, method, scheme, host, port, path, httpversion, headers, content,
                           timestamp_start, timestamp_end)
Beispiel #5
0
    def read_request_proxy(self, client_conn):
        line = self.get_line(self.rfile)
        if line == "":
            return None

        if not self.proxy_connect_state:
            connparts = http.parse_init_connect(line)
            if connparts:
                host, port, httpversion = connparts
                headers = self.read_headers(authenticate=True)
                self.wfile.write(
                            'HTTP/1.1 200 Connection established\r\n' +
                            ('Proxy-agent: %s\r\n'%self.server_version) +
                            '\r\n'
                            )
                self.wfile.flush()
                self.establish_ssl(client_conn, host, port)
                self.proxy_connect_state = (host, port, httpversion)
                line = self.rfile.readline(line)

        if self.proxy_connect_state:
            r = http.parse_init_http(line)
            if not r:
                raise ProxyError(400, "Bad HTTP request line: %s"%repr(line))
            method, path, httpversion = r
            headers = self.read_headers(authenticate=False)

            host, port, _ = self.proxy_connect_state
            content = http.read_http_body_request(
                self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
            )
            return flow.Request(
                        client_conn, httpversion, host, port, "https", method, path, headers, content,
                        self.rfile.first_byte_timestamp, utils.timestamp()
                   )
        else:
            r = http.parse_init_proxy(line)
            if not r:
                raise ProxyError(400, "Bad HTTP request line: %s"%repr(line))
            method, scheme, host, port, path, httpversion = r
            headers = self.read_headers(authenticate=True)
            content = http.read_http_body_request(
                self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
            )
            return flow.Request(
                        client_conn, httpversion, host, port, scheme, method, path, headers, content,
                        self.rfile.first_byte_timestamp, utils.timestamp()
                    )
Beispiel #6
0
 def _read_request_absolute_form(self, client_conn, line):
     """
     When making a request to a proxy (other than CONNECT or OPTIONS),
     a client must send the target uri in absolute-form.
     An example absolute-form request line would be:
         GET http://www.example.com/foo.html HTTP/1.1
     """
     r = http.parse_init_proxy(line)
     if not r:
         raise ProxyError(400, "Bad HTTP request line: %s" % repr(line))
     method, scheme, host, port, path, httpversion = r
     headers = self.read_headers(authenticate=True)
     content = http.read_http_body_request(self.rfile, self.wfile, headers,
                                           httpversion,
                                           self.config.body_size_limit)
     return flow.Request(client_conn, httpversion, host, port, scheme,
                         method, path, headers, content,
                         self.rfile.first_byte_timestamp, utils.timestamp())
Beispiel #7
0
 def _read_request_absolute_form(self, client_conn, line):
     """
     When making a request to a proxy (other than CONNECT or OPTIONS),
     a client must send the target uri in absolute-form.
     An example absolute-form request line would be:
         GET http://www.example.com/foo.html HTTP/1.1
     """
     r = http.parse_init_proxy(line)
     if not r:
         raise ProxyError(400, "Bad HTTP request line: %s"%repr(line))
     method, scheme, host, port, path, httpversion = r
     headers = self.read_headers(authenticate=True)
     content = http.read_http_body_request(
         self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
     )
     return flow.Request(
         client_conn, httpversion, host, port, scheme, method, path, headers, content,
         self.rfile.first_byte_timestamp, utils.timestamp()
     )
Beispiel #8
0
                    raise ProxyError(400, str(v))
                self.proxy_connect_state = (host, port, httpversion)
                line = self.rfile.readline(line)
            if self.proxy_connect_state:
                host, port, httpversion = self.proxy_connect_state
                r = http.parse_init_http(line)
                if not r:
                    raise ProxyError(400, "Bad HTTP request line: %s"%repr(line))
                method, path, httpversion = r
                headers = http.read_headers(self.rfile)
                content = http.read_http_body_request(
                    self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
                )
                return flow.Request(client_conn, httpversion, host, port, "https", method, path, headers, content)
            else:
                r = http.parse_init_proxy(line)
                if not r:
                    raise ProxyError(400, "Bad HTTP request line: %s"%repr(line))
                method, scheme, host, port, path, httpversion = http.parse_init_proxy(line)
                headers = http.read_headers(self.rfile)
                content = http.read_http_body_request(
                    self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
                )
                return flow.Request(client_conn, httpversion, host, port, scheme, method, path, headers, content)

    def send_response(self, response):
        d = response._assemble()
        if not d:
            raise ProxyError(502, "Incomplete response could not not be readied for transmission.")
        self.wfile.write(d)
        self.wfile.flush()
Beispiel #9
0
            r = http.parse_init_http(line)
            if not r:
                raise ProxyError(400, "Bad HTTP request line: %s"%repr(line))
            method, path, httpversion = r
            headers = self.read_headers(authenticate=False)

            host, port, _ = self.proxy_connect_state
            content = http.read_http_body_request(
                self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
            )
            return flow.Request(
                        client_conn, httpversion, host, port, "https", method, path, headers, content,
                        self.rfile.first_byte_timestamp, utils.timestamp()
                   )
        else:
            r = http.parse_init_proxy(line)
            if not r:
                raise ProxyError(400, "Bad HTTP request line: %s"%repr(line))
            method, scheme, host, port, path, httpversion = r
            headers = self.read_headers(authenticate=True)
            content = http.read_http_body_request(
                self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
            )
            return flow.Request(
                        client_conn, httpversion, host, port, scheme, method, path, headers, content,
                        self.rfile.first_byte_timestamp, utils.timestamp()
                    )

    def read_request_reverse(self, client_conn):
        line = self.get_line(self.rfile)
        if line == "":
Beispiel #10
0
class PathodHandler(tcp.BaseHandler):
    wbufsize = 0
    sni = None

    def info(self, s):
        logger.info("%s:%s: %s" %
                    (self.address.host, self.address.port, str(s)))

    def handle_sni(self, connection):
        self.sni = connection.get_servername()

    def serve_crafted(self, crafted):
        c = self.server.check_policy(crafted, self.server.request_settings)
        if c:
            err = language.make_error_response(c)
            language.serve(err, self.wfile, self.server.request_settings)
            log = dict(type="error", msg=c)
            return False, log

        if self.server.explain and not isinstance(
                crafted, language.PathodErrorResponse):
            crafted = crafted.freeze(self.server.request_settings, None)
            self.info(">> Spec: %s" % crafted.spec())
        response_log = language.serve(crafted, self.wfile,
                                      self.server.request_settings, None)
        if response_log["disconnect"]:
            return False, response_log
        return True, response_log

    def handle_request(self):
        """
            Returns a (again, log) tuple.

            again: True if request handling should continue.
            log: A dictionary, or None
        """
        line = self.rfile.readline()
        if line == "\r\n" or line == "\n":  # Possible leftover from previous message
            line = self.rfile.readline()
        if line == "":
            # Normal termination
            return False, None

        m = utils.MemBool()
        if m(http.parse_init_connect(line)):
            headers = http.read_headers(self.rfile)
            self.wfile.write('HTTP/1.1 200 Connection established\r\n' +
                             ('Proxy-agent: %s\r\n' % version.NAMEVERSION) +
                             '\r\n')
            self.wfile.flush()
            if not self.server.ssloptions.not_after_connect:
                try:
                    cert, key, chain_file = self.server.ssloptions.get_cert(
                        m.v[0])
                    self.convert_to_ssl(
                        cert,
                        key,
                        handle_sni=self.handle_sni,
                        request_client_cert=self.server.ssloptions.
                        request_client_cert,
                        cipher_list=self.server.ssloptions.ciphers,
                        method=self.server.ssloptions.sslversion,
                    )
                except tcp.NetLibError, v:
                    s = str(v)
                    self.info(s)
                    return False, dict(type="error", msg=s)
            return True, None
        elif m(http.parse_init_proxy(line)):
            method, _, _, _, path, httpversion = m.v
Beispiel #11
0
 def read_request(self, client_conn):
     if self.config.transparent_proxy:
         host, port = self.config.transparent_proxy["resolver"].original_addr(self.connection)
         if not self.ssl_established and (port in self.config.transparent_proxy["sslports"]):
             scheme = "https"
             certfile = self.find_cert(host, port, None)
             self.convert_to_ssl(certfile, self.config.certfile or self.config.cacert)
         else:
             scheme = "http"
         host = self.sni or host
         line = self.get_line(self.rfile)
         if line == "":
             return None
         r = http.parse_init_http(line)
         if not r:
             raise ProxyError(400, "Bad HTTP request line: %s"%line)
         method, path, httpversion = r
         headers = http.read_headers(self.rfile)
         content = http.read_http_body_request(
                     self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
                 )
         return flow.Request(client_conn, httpversion, host, port, scheme, method, path, headers, content)
     elif self.config.reverse_proxy:
         line = self.get_line(self.rfile)
         if line == "":
             return None
         scheme, host, port = self.config.reverse_proxy
         r = http.parse_init_http(line)
         if not r:
             raise ProxyError(400, "Bad HTTP request line: %s"%line)
         method, path, httpversion = r
         headers = http.read_headers(self.rfile)
         content = http.read_http_body_request(
                     self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
                 )
         return flow.Request(client_conn, httpversion, host, port, "http", method, path, headers, content)
     else:
         line = self.get_line(self.rfile)
         if line == "":
             return None
         if line.startswith("CONNECT"):
             host, port, httpversion = http.parse_init_connect(line)
             # FIXME: Discard additional headers sent to the proxy. Should I expose
             # these to users?
             while 1:
                 d = self.rfile.readline()
                 if d == '\r\n' or d == '\n':
                     break
             self.wfile.write(
                         'HTTP/1.1 200 Connection established\r\n' +
                         ('Proxy-agent: %s\r\n'%version.NAMEVERSION) +
                         '\r\n'
                         )
             self.wfile.flush()
             certfile = self.find_cert(host, port, None)
             self.convert_to_ssl(certfile, self.config.certfile or self.config.cacert)
             self.proxy_connect_state = (host, port, httpversion)
             line = self.rfile.readline(line)
         if self.proxy_connect_state:
             host, port, httpversion = self.proxy_connect_state
             r = http.parse_init_http(line)
             if not r:
                 raise ProxyError(400, "Bad HTTP request line: %s"%line)
             method, path, httpversion = r
             headers = http.read_headers(self.rfile)
             content = http.read_http_body_request(
                 self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
             )
             return flow.Request(client_conn, httpversion, host, port, "https", method, path, headers, content)
         else:
             method, scheme, host, port, path, httpversion = http.parse_init_proxy(line)
             headers = http.read_headers(self.rfile)
             content = http.read_http_body_request(
                 self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
             )
             return flow.Request(client_conn, httpversion, host, port, scheme, method, path, headers, content)
Beispiel #12
0
            r = http.parse_init_http(line)
            if not r:
                raise ProxyError(400, "Bad HTTP request line: %s" % repr(line))
            method, path, httpversion = r
            headers = self.read_headers(authenticate=False)

            host, port, _ = self.proxy_connect_state
            content = http.read_http_body_request(self.rfile, self.wfile,
                                                  headers, httpversion,
                                                  self.config.body_size_limit)
            return flow.Request(client_conn, httpversion, host, port, "https",
                                method, path, headers, content,
                                self.rfile.first_byte_timestamp,
                                utils.timestamp())
        else:
            r = http.parse_init_proxy(line)
            if not r:
                raise ProxyError(400, "Bad HTTP request line: %s" % repr(line))
            method, scheme, host, port, path, httpversion = r
            headers = self.read_headers(authenticate=True)
            content = http.read_http_body_request(self.rfile, self.wfile,
                                                  headers, httpversion,
                                                  self.config.body_size_limit)
            return flow.Request(client_conn, httpversion, host, port, scheme,
                                method, path, headers, content,
                                self.rfile.first_byte_timestamp,
                                utils.timestamp())

    def read_request_reverse(self, client_conn):
        line = self.get_line(self.rfile)
        if line == "":
Beispiel #13
0
    def handle_request(self):
        """
            Returns a (again, log) tuple.

            again: True if request handling should continue.
            log: A dictionary, or None
        """
        line = self.rfile.readline()
        if line == "\r\n" or line == "\n":  # Possible leftover from previous message
            line = self.rfile.readline()
        if line == "":
            # Normal termination
            return False, None

        m = utils.MemBool()
        if m(http.parse_init_connect(line)):
            headers = http.read_headers(self.rfile)
            self.wfile.write(
                'HTTP/1.1 200 Connection established\r\n' +
                ('Proxy-agent: %s\r\n' % version.NAMEVERSION) +
                '\r\n'
            )
            self.wfile.flush()
            if not self.server.ssloptions.not_after_connect:
                try:
                    cert, key, chain_file = self.server.ssloptions.get_cert(m.v[0])
                    self.convert_to_ssl(
                        cert, key,
                        handle_sni=self.handle_sni,
                        request_client_cert=self.server.ssloptions.request_client_cert,
                        cipher_list=self.server.ssloptions.ciphers,
                        method=self.server.ssloptions.sslversion,
                    )
                except tcp.NetLibError as v:
                    s = str(v)
                    self.info(s)
                    return False, dict(type="error", msg=s)
            return True, None
        elif m(http.parse_init_proxy(line)):
            method, _, _, _, path, httpversion = m.v
        elif m(http.parse_init_http(line)):
            method, path, httpversion = m.v
        else:
            s = "Invalid first line: %s" % repr(line)
            self.info(s)
            return False, dict(type="error", msg=s)

        headers = http.read_headers(self.rfile)
        if headers is None:
            s = "Invalid headers"
            self.info(s)
            return False, dict(type="error", msg=s)

        clientcert = None
        if self.clientcert:
            clientcert = dict(
                cn=self.clientcert.cn,
                subject=self.clientcert.subject,
                serial=self.clientcert.serial,
                notbefore=self.clientcert.notbefore.isoformat(),
                notafter=self.clientcert.notafter.isoformat(),
                keyinfo=self.clientcert.keyinfo,
            )

        retlog = dict(
            type="crafted",
            request=dict(
                path=path,
                method=method,
                headers=headers.lst,
                httpversion=httpversion,
                sni=self.sni,
                remote_address=self.address(),
                clientcert=clientcert,
            ),
            cipher=None,
        )
        if self.ssl_established:
            retlog["cipher"] = self.get_current_cipher()

        try:
            content = http.read_http_body(
                self.rfile, headers, None,
                method, None, True
            )
        except http.HttpError as s:
            s = str(s)
            self.info(s)
            return False, dict(type="error", msg=s)

        for i in self.server.anchors:
            if i[0].match(path):
                self.info("crafting anchor: %s" % path)
                again, retlog["response"] = self.serve_crafted(i[1])
                return again, retlog

        if not self.server.nocraft and path.startswith(self.server.craftanchor):
            spec = urllib.parse.unquote(path)[len(self.server.craftanchor):]
            self.info("crafting spec: %s" % spec)
            try:
                crafted = language.parse_response(spec)
            except language.ParseException as v:
                self.info("Parse error: %s" % v.msg)
                crafted = language.make_error_response(
                    "Parse Error",
                    "Error parsing response spec: %s\n" % v.msg + v.marked()
                )
            again, retlog["response"] = self.serve_crafted(crafted)
            return again, retlog
        elif self.server.noweb:
            crafted = language.make_error_response("Access Denied")
            language.serve(crafted, self.wfile, self.server.request_settings)
            return False, dict(
                type="error",
                msg="Access denied: web interface disabled"
            )
        else:
            self.info("app: %s %s" % (method, path))
            req = wsgi.Request("http", method, path, headers, content)
            flow = wsgi.Flow(self.address, req)
            sn = self.connection.getsockname()
            a = wsgi.WSGIAdaptor(
                self.server.app,
                sn[0],
                self.server.address.port,
                version.NAMEVERSION
            )
            a.serve(flow, self.wfile)
            return True, None
Beispiel #14
0
                line = self.rfile.readline(line)
            if self.proxy_connect_state:
                host, port, httpversion = self.proxy_connect_state
                r = http.parse_init_http(line)
                if not r:
                    raise ProxyError(400, "Bad HTTP request line: %s"%repr(line))
                method, path, httpversion = r
                headers = http.read_headers(self.rfile)
                if headers is None:
                    raise ProxyError(400, "Invalid headers")
                content = http.read_http_body_request(
                    self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
                )
                return flow.Request(client_conn, httpversion, host, port, "https", method, path, headers, content)
            else:
                r = http.parse_init_proxy(line)
                if not r:
                    raise ProxyError(400, "Bad HTTP request line: %s"%repr(line))
                method, scheme, host, port, path, httpversion = http.parse_init_proxy(line)
                headers = http.read_headers(self.rfile)
                if headers is None:
                    raise ProxyError(400, "Invalid headers")
                content = http.read_http_body_request(
                    self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
                )
                return flow.Request(client_conn, httpversion, host, port, scheme, method, path, headers, content)

    def send_response(self, response):
        d = response._assemble()
        if not d:
            raise ProxyError(502, "Incomplete response could not not be readied for transmission.")