def http_connect(self, connect_to): self.wfile.write( 'CONNECT %s:%s HTTP/1.1\r\n'%tuple(connect_to) + '\r\n' ) self.wfile.flush() l = self.rfile.readline() if not l: raise PathocError("Proxy CONNECT failed") parsed = http.parse_response_line(l) if not parsed[1] == 200: raise PathocError("Proxy CONNECT failed: %s - %s"%(parsed[1], parsed[2])) http.read_headers(self.rfile)
def create_http_request(flowheader, reqbuf): sfp = StringIO(reqbuf) method, url, httpversion = http.parse_init_http(sfp.readline()) host, port, path = parse_url(url) headers = http.read_headers(sfp) if not host: if not headers.get("host"): host = flowheader.dstip else: host = headers.get("host")[0] if ":" in host: host = string.rsplit(host, ":", maxsplit=1)[0] if port == None: port = flowheader.dport # TODO: passing None as the second arg will produce and error if "expect" is in the headers content = http.read_http_body_request(sfp, None, headers, httpversion, None) # content = http.read_http_body(sfp, headers, True, None) return flow.Request( None, httpversion, host, port, "http", method, path, headers, content, flowheader.ts_request_start, flowheader.ts_request_finish, )
def handle_request(self): """ Returns a (again, log) tuple. again: True if request handling should continue. log: A dictionary, or None """ line = self.rfile.readline() if line == "\r\n" or line == "\n": # Possible leftover from previous message line = self.rfile.readline() if line == "": # Normal termination return False, None m = utils.MemBool() if m(http.parse_init_connect(line)): headers = http.read_headers(self.rfile) self.wfile.write( 'HTTP/1.1 200 Connection established\r\n' + ('Proxy-agent: %s\r\n'%version.NAMEVERSION) + '\r\n' ) self.wfile.flush() if not self.server.ssloptions.not_after_connect: try: self.convert_to_ssl( self.server.ssloptions.certfile, self.server.ssloptions.keyfile, ) except tcp.NetLibError, v: s = str(v) self.info(s) return False, dict(type = "error", msg = s) return True, None
def read_request(self, client_conn): if self.config.transparent_proxy: host, port = self.config.transparent_proxy["resolver"].original_addr(self.connection) if not self.ssl_established and (port in self.config.transparent_proxy["sslports"]): scheme = "https" certfile = self.find_cert(host, port, None) try: self.convert_to_ssl(certfile, self.config.certfile or self.config.cacert) except tcp.NetLibError, v: raise ProxyError(400, str(v)) else: scheme = "http" host = self.sni or host line = self.get_line(self.rfile) if line == "": return None r = http.parse_init_http(line) if not r: raise ProxyError(400, "Bad HTTP request line: %s"%repr(line)) method, path, httpversion = r headers = http.read_headers(self.rfile) content = http.read_http_body_request( self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit ) return flow.Request(client_conn, httpversion, host, port, scheme, method, path, headers, content)
def http_connect(self, connect_to, wfile, rfile): wfile.write( 'CONNECT %s:%s HTTP/1.1\r\n'%tuple(connect_to) + '\r\n' ) wfile.flush() rfile.readline() headers = http.read_headers(self.rfile)
def from_stream(cls, rfile, include_content=True, body_size_limit=None): """ Parse an HTTP request from a file stream """ httpversion, host, port, scheme, method, path, headers, content, timestamp_start, timestamp_end \ = None, None, None, None, None, None, None, None, None, None if hasattr(rfile, "reset_timestamps"): rfile.reset_timestamps() request_line = get_line(rfile) if hasattr(rfile, "first_byte_timestamp"): timestamp_start = rfile.first_byte_timestamp else: timestamp_start = utils.timestamp() request_line_parts = http.parse_init(request_line) if not request_line_parts: raise http.HttpError( 400, "Bad HTTP request line: %s" % repr(request_line)) method, path, httpversion = request_line_parts if path == '*': form_in = "asterisk" elif path.startswith("/"): form_in = "origin" if not netlib.utils.isascii(path): raise http.HttpError( 400, "Bad HTTP request line: %s" % repr(request_line)) elif method.upper() == 'CONNECT': form_in = "authority" r = http.parse_init_connect(request_line) if not r: raise http.HttpError( 400, "Bad HTTP request line: %s" % repr(request_line)) host, port, _ = r path = None else: form_in = "absolute" r = http.parse_init_proxy(request_line) if not r: raise http.HttpError( 400, "Bad HTTP request line: %s" % repr(request_line)) _, scheme, host, port, path, _ = r headers = http.read_headers(rfile) if headers is None: raise http.HttpError(400, "Invalid headers") if include_content: content = http.read_http_body(rfile, headers, body_size_limit, True) timestamp_end = utils.timestamp() return HTTPRequest(form_in, method, scheme, host, port, path, httpversion, headers, content, timestamp_start, timestamp_end)
def handle_request(self): """ Returns True if handling should continue. """ line = self.rfile.readline() if line == "\r\n" or line == "\n": # Possible leftover from previous message line = self.rfile.readline() if line == "": return parts = http.parse_init_http(line) if not parts: s = "Invalid first line: %s"%repr(line) self.info(s) self.server.add_log( dict( type = "error", msg = s ) ) return method, path, httpversion = parts headers = http.read_headers(self.rfile) if headers is None: s = "Invalid headers" self.info(s) self.server.add_log( dict( type = "error", msg = s ) ) return request_log = dict( path = path, method = method, headers = headers.lst, httpversion = httpversion, sni = self.sni, remote_address = self.client_address, ) try: content = http.read_http_body_request( self.rfile, self.wfile, headers, httpversion, None ) except http.HttpError, s: s = str(s) self.info(s) self.server.add_log( dict( type = "error", msg = s ) ) return
def test_read_multi(self): data = """ Header: one Header: two \r\n """ data = textwrap.dedent(data) data = data.strip() s = cStringIO.StringIO(data) h = http.read_headers(s) assert h.lst == [["Header", "one"], ["Header", "two"]]
def test_read_continued(self): data = """ Header: one \ttwo Header2: three \r\n """ data = textwrap.dedent(data) data = data.strip() s = cStringIO.StringIO(data) h = http.read_headers(s) assert h.lst == [["Header", "one\r\n two"], ["Header2", "three"]]
def read_headers(self, authenticate=False): headers = http.read_headers(self.rfile) if headers is None: raise ProxyError(400, "Invalid headers") if authenticate and self.config.authenticator: if self.config.authenticator.authenticate(headers): self.config.authenticator.clean(headers) else: raise ProxyError( 407, "Proxy Authentication Required", self.config.authenticator.auth_challenge_headers()) return headers
def from_stream(cls, rfile, include_content=True, body_size_limit=None): """ Parse an HTTP request from a file stream """ httpversion, host, port, scheme, method, path, headers, content, timestamp_start, timestamp_end \ = None, None, None, None, None, None, None, None, None, None if hasattr(rfile, "reset_timestamps"): rfile.reset_timestamps() request_line = get_line(rfile) if hasattr(rfile, "first_byte_timestamp"): timestamp_start = rfile.first_byte_timestamp else: timestamp_start = utils.timestamp() request_line_parts = http.parse_init(request_line) if not request_line_parts: raise http.HttpError(400, "Bad HTTP request line: %s" % repr(request_line)) method, path, httpversion = request_line_parts if path == '*': form_in = "asterisk" elif path.startswith("/"): form_in = "origin" if not netlib.utils.isascii(path): raise http.HttpError(400, "Bad HTTP request line: %s" % repr(request_line)) elif method.upper() == 'CONNECT': form_in = "authority" r = http.parse_init_connect(request_line) if not r: raise http.HttpError(400, "Bad HTTP request line: %s" % repr(request_line)) host, port, _ = r path = None else: form_in = "absolute" r = http.parse_init_proxy(request_line) if not r: raise http.HttpError(400, "Bad HTTP request line: %s" % repr(request_line)) _, scheme, host, port, path, _ = r headers = http.read_headers(rfile) if headers is None: raise http.HttpError(400, "Invalid headers") if include_content: content = http.read_http_body(rfile, headers, body_size_limit, True) timestamp_end = utils.timestamp() return HTTPRequest(form_in, method, scheme, host, port, path, httpversion, headers, content, timestamp_start, timestamp_end)
def read_headers(self, authenticate=False): headers = http.read_headers(self.rfile) if headers is None: raise ProxyError(400, "Invalid headers") if authenticate and self.config.authenticator: if self.config.authenticator.authenticate(headers): self.config.authenticator.clean(headers) else: raise ProxyError( 407, "Proxy Authentication Required", self.config.authenticator.auth_challenge_headers() ) return headers
def handle_request(self): """ Returns a (again, log) tuple. again: True if request handling should continue. log: A dictionary, or None """ if self.server.logreq: self.rfile.start_log() if self.server.logresp: self.wfile.start_log() line = http.get_request_line(self.rfile) if not line: # Normal termination return False m = utils.MemBool() if m(http.parse_init_connect(line)): headers = http.read_headers(self.rfile) self.wfile.write( 'HTTP/1.1 200 Connection established\r\n' + ('Proxy-agent: %s\r\n' % version.NAMEVERSION) + '\r\n' ) self.wfile.flush() if not self.server.ssloptions.not_after_connect: try: cert, key, chain_file = self.server.ssloptions.get_cert( m.v[0] ) self.convert_to_ssl( cert, key, handle_sni=self.handle_sni, request_client_cert=self.server.ssloptions.request_client_cert, cipher_list=self.server.ssloptions.ciphers, method=self.server.ssloptions.sslversion, ) except tcp.NetLibError, v: s = str(v) self.info(s) self.addlog(dict(type="error", msg=s)) return False return True
def handle_request(self): """ Returns a (again, log) tuple. again: True if request handling should continue. log: A dictionary, or None """ line = self.rfile.readline() if line == "\r\n" or line == "\n": # Possible leftover from previous message line = self.rfile.readline() if line == "": # Normal termination return False, None parts = http.parse_init_http(line) if not parts: s = "Invalid first line: %s"%repr(line) self.info(s) return False, dict(type = "error", msg = s) method, path, httpversion = parts headers = http.read_headers(self.rfile) if headers is None: s = "Invalid headers" self.info(s) return False, dict(type = "error", msg = s) request_log = dict( path = path, method = method, headers = headers.lst, httpversion = httpversion, sni = self.sni, remote_address = self.client_address, ) try: content = http.read_http_body_request( self.rfile, self.wfile, headers, httpversion, None ) except http.HttpError, s: s = str(s) self.info(s) return False, dict(type = "error", msg = s)
def handle_request(self): """ Returns a (again, log) tuple. again: True if request handling should continue. log: A dictionary, or None """ line = self.rfile.readline() if line == "\r\n" or line == "\n": # Possible leftover from previous message line = self.rfile.readline() if line == "": # Normal termination return False, None parts = http.parse_init_http(line) if not parts: s = "Invalid first line: %s" % repr(line) self.info(s) return False, dict(type="error", msg=s) method, path, httpversion = parts headers = http.read_headers(self.rfile) if headers is None: s = "Invalid headers" self.info(s) return False, dict(type="error", msg=s) request_log = dict( path=path, method=method, headers=headers.lst, httpversion=httpversion, sni=self.sni, remote_address=self.client_address, ) try: content = http.read_http_body_request(self.rfile, self.wfile, headers, httpversion, None) except http.HttpError, s: s = str(s) self.info(s) return False, dict(type="error", msg=s)
def handle_request(self): """ Returns a (again, log) tuple. again: True if request handling should continue. log: A dictionary, or None """ if self.server.logreq: self.rfile.start_log() if self.server.logresp: self.wfile.start_log() line = http.get_request_line(self.rfile) if not line: # Normal termination return False m = utils.MemBool() if m(http.parse_init_connect(line)): headers = http.read_headers(self.rfile) self.wfile.write('HTTP/1.1 200 Connection established\r\n' + ('Proxy-agent: %s\r\n' % version.NAMEVERSION) + '\r\n') self.wfile.flush() if not self.server.ssloptions.not_after_connect: try: cert, key, chain_file = self.server.ssloptions.get_cert( m.v[0]) self.convert_to_ssl( cert, key, handle_sni=self.handle_sni, request_client_cert=self.server.ssloptions. request_client_cert, cipher_list=self.server.ssloptions.ciphers, method=self.server.ssloptions.sslversion, ) except tcp.NetLibError, v: s = str(v) self.info(s) self.addlog(dict(type="error", msg=s)) return False return True
def read_request(self, client_conn): if self.config.transparent_proxy: host, port = self.config.transparent_proxy["resolver"].original_addr(self.connection) if not self.ssl_established and (port in self.config.transparent_proxy["sslports"]): scheme = "https" certfile = self.find_cert(host, port, None) self.convert_to_ssl(certfile, self.config.certfile or self.config.cacert) else: scheme = "http" host = self.sni or host line = self.get_line(self.rfile) if line == "": return None r = http.parse_init_http(line) if not r: raise ProxyError(400, "Bad HTTP request line: %s"%line) method, path, httpversion = r headers = http.read_headers(self.rfile) content = http.read_http_body_request( self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit ) return flow.Request(client_conn, httpversion, host, port, scheme, method, path, headers, content) elif self.config.reverse_proxy: line = self.get_line(self.rfile) if line == "": return None scheme, host, port = self.config.reverse_proxy r = http.parse_init_http(line) if not r: raise ProxyError(400, "Bad HTTP request line: %s"%line) method, path, httpversion = r headers = http.read_headers(self.rfile) content = http.read_http_body_request( self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit ) return flow.Request(client_conn, httpversion, host, port, "http", method, path, headers, content) else: line = self.get_line(self.rfile) if line == "": return None if line.startswith("CONNECT"): host, port, httpversion = http.parse_init_connect(line) # FIXME: Discard additional headers sent to the proxy. Should I expose # these to users? while 1: d = self.rfile.readline() if d == '\r\n' or d == '\n': break self.wfile.write( 'HTTP/1.1 200 Connection established\r\n' + ('Proxy-agent: %s\r\n'%version.NAMEVERSION) + '\r\n' ) self.wfile.flush() certfile = self.find_cert(host, port, None) self.convert_to_ssl(certfile, self.config.certfile or self.config.cacert) self.proxy_connect_state = (host, port, httpversion) line = self.rfile.readline(line) if self.proxy_connect_state: host, port, httpversion = self.proxy_connect_state r = http.parse_init_http(line) if not r: raise ProxyError(400, "Bad HTTP request line: %s"%line) method, path, httpversion = r headers = http.read_headers(self.rfile) content = http.read_http_body_request( self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit ) return flow.Request(client_conn, httpversion, host, port, "https", method, path, headers, content) else: method, scheme, host, port, path, httpversion = http.parse_init_proxy(line) headers = http.read_headers(self.rfile) content = http.read_http_body_request( self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit ) return flow.Request(client_conn, httpversion, host, port, scheme, method, path, headers, content)
method, path, httpversion = r headers = http.read_headers(self.rfile) content = http.read_http_body_request( self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit ) return flow.Request(client_conn, httpversion, host, port, scheme, method, path, headers, content) elif self.config.reverse_proxy: line = self.get_line(self.rfile) if line == "": return None scheme, host, port = self.config.reverse_proxy r = http.parse_init_http(line) if not r: raise ProxyError(400, "Bad HTTP request line: %s"%repr(line)) method, path, httpversion = r headers = http.read_headers(self.rfile) content = http.read_http_body_request( self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit ) return flow.Request(client_conn, httpversion, host, port, "http", method, path, headers, content) else: line = self.get_line(self.rfile) if line == "": return None if line.startswith("CONNECT"): r = http.parse_init_connect(line) if not r: raise ProxyError(400, "Bad HTTP request line: %s"%repr(line)) host, port, httpversion = r # FIXME: Discard additional headers sent to the proxy. Should I expose # these to users?
def _read(self, data, verbatim=False): if not verbatim: data = textwrap.dedent(data) data = data.strip() s = cStringIO.StringIO(data) return http.read_headers(s)
) except tcp.NetLibError, v: s = str(v) self.info(s) return False, dict(type="error", msg=s) return True, None elif m(http.parse_init_proxy(line)): method, _, _, _, path, httpversion = m.v elif m(http.parse_init_http(line)): method, path, httpversion = m.v else: s = "Invalid first line: %s" % repr(line) self.info(s) return False, dict(type="error", msg=s) headers = http.read_headers(self.rfile) if headers is None: s = "Invalid headers" self.info(s) return False, dict(type="error", msg=s) clientcert = None if self.clientcert: clientcert = dict( cn=self.clientcert.cn, subject=self.clientcert.subject, serial=self.clientcert.serial, notbefore=self.clientcert.notbefore.isoformat(), notafter=self.clientcert.notafter.isoformat(), keyinfo=self.clientcert.keyinfo, )
def handle_request(self): """ Returns a (again, log) tuple. again: True if request handling should continue. log: A dictionary, or None """ line = self.rfile.readline() if line == "\r\n" or line == "\n": # Possible leftover from previous message line = self.rfile.readline() if line == "": # Normal termination return False, None m = utils.MemBool() if m(http.parse_init_connect(line)): headers = http.read_headers(self.rfile) self.wfile.write( 'HTTP/1.1 200 Connection established\r\n' + ('Proxy-agent: %s\r\n' % version.NAMEVERSION) + '\r\n' ) self.wfile.flush() if not self.server.ssloptions.not_after_connect: try: cert, key, chain_file = self.server.ssloptions.get_cert(m.v[0]) self.convert_to_ssl( cert, key, handle_sni=self.handle_sni, request_client_cert=self.server.ssloptions.request_client_cert, cipher_list=self.server.ssloptions.ciphers, method=self.server.ssloptions.sslversion, ) except tcp.NetLibError as v: s = str(v) self.info(s) return False, dict(type="error", msg=s) return True, None elif m(http.parse_init_proxy(line)): method, _, _, _, path, httpversion = m.v elif m(http.parse_init_http(line)): method, path, httpversion = m.v else: s = "Invalid first line: %s" % repr(line) self.info(s) return False, dict(type="error", msg=s) headers = http.read_headers(self.rfile) if headers is None: s = "Invalid headers" self.info(s) return False, dict(type="error", msg=s) clientcert = None if self.clientcert: clientcert = dict( cn=self.clientcert.cn, subject=self.clientcert.subject, serial=self.clientcert.serial, notbefore=self.clientcert.notbefore.isoformat(), notafter=self.clientcert.notafter.isoformat(), keyinfo=self.clientcert.keyinfo, ) retlog = dict( type="crafted", request=dict( path=path, method=method, headers=headers.lst, httpversion=httpversion, sni=self.sni, remote_address=self.address(), clientcert=clientcert, ), cipher=None, ) if self.ssl_established: retlog["cipher"] = self.get_current_cipher() try: content = http.read_http_body( self.rfile, headers, None, method, None, True ) except http.HttpError as s: s = str(s) self.info(s) return False, dict(type="error", msg=s) for i in self.server.anchors: if i[0].match(path): self.info("crafting anchor: %s" % path) again, retlog["response"] = self.serve_crafted(i[1]) return again, retlog if not self.server.nocraft and path.startswith(self.server.craftanchor): spec = urllib.parse.unquote(path)[len(self.server.craftanchor):] self.info("crafting spec: %s" % spec) try: crafted = language.parse_response(spec) except language.ParseException as v: self.info("Parse error: %s" % v.msg) crafted = language.make_error_response( "Parse Error", "Error parsing response spec: %s\n" % v.msg + v.marked() ) again, retlog["response"] = self.serve_crafted(crafted) return again, retlog elif self.server.noweb: crafted = language.make_error_response("Access Denied") language.serve(crafted, self.wfile, self.server.request_settings) return False, dict( type="error", msg="Access denied: web interface disabled" ) else: self.info("app: %s %s" % (method, path)) req = wsgi.Request("http", method, path, headers, content) flow = wsgi.Flow(self.address, req) sn = self.connection.getsockname() a = wsgi.WSGIAdaptor( self.server.app, sn[0], self.server.address.port, version.NAMEVERSION ) a.serve(flow, self.wfile) return True, None