def recv(self, wsgi=False): try: stream = HttpStream(self.reader, kind=HTTP_REQUEST, parser_class=HttpParser, decompress=True) if bool(wsgi): environ = stream.wsgi_environ() environ['wsgi.url_scheme'] = guess_scheme(environ) environ['wsgi.input'] = stream.body_file() environ['wsgi.socket'] = self.socket return environ # BUG: # http-parser has an issue here, if we call 'method' before 'headers' # and invalid method name is returned... fields = stream.headers() method = stream.method() url = stream.url() version = stream.version() content = stream.body_file() url = urlparse(url) path = url.path query = parse_qs(url.query, keep_blank_values=True) fragment = url.fragment for k, v in six.iteritems(dict(query)): query[k] = parse_query_value(v) self.version = 'HTTP/%s.%s' % version return method, path, query, fragment, fields, content except NoMoreData: pass
def get_response(self, request, connection): """ return final respons, it is only accessible via peform method """ if log.isEnabledFor(logging.DEBUG): log.debug("Start to parse response") p = HttpStream(SocketReader(connection.socket()), kind=1, decompress=self.decompress) if log.isEnabledFor(logging.DEBUG): log.debug("Got response: %s %s" % (p.version(), p.status())) log.debug("headers: [%s]" % p.headers()) location = p.headers().get('location') if self.follow_redirect: should_close = not p.should_keep_alive() if p.status_code() in (301, 302, 307,): # read full body and release the connection p.body_file().read() connection.release(should_close) if request.method in ('GET', 'HEAD',) or \ self.force_follow_redirect: if hasattr(self.body, 'read'): try: self.body.seek(0) except AttributeError: raise RequestError("Can't redirect %s to %s " "because body has already been read" % (self.url, location)) return self.redirect(location, request) elif p.status_code() == 303 and self.method == "POST": # read full body and release the connection p.body_file().read() connection.release(should_close) request.method = "GET" request.body = None return self.redirect(location, request) # create response object resp = self.response_class(connection, request, p) # apply response filters for f in self.response_filters: f.on_response(resp, request) if log.isEnabledFor(logging.DEBUG): log.debug("return response class") # return final response return resp
def recv(self): try: stream = HttpStream(self.reader, kind=HTTP_RESPONSE, parser_class=HttpParser, decompress=True) status = stream.status_code() version = stream.version() fields = stream.headers() content = stream.body_file() self.version = 'HTTP/%s.%s' % version return status, fields, content except NoMoreData: pass
def join_data(request: HttpStream, type_: str): assert type_ in ('Request', 'Response') # try: major, minor = request.version() # except ParserError: # raise if type_ == 'Request': url = request.url() start_line = f'{request.method()} {url} HTTP/{major}.{minor}' else: start_line = f'HTTP/{major}.{minor} {request.status()}' body = request.body_string(binary=True) headers = request.headers() setcookies = headers['Set-Cookie'] if 'Set-Cookie' in headers else None if "Content-Encoding" in headers: assert headers["Content-Encoding"] == "br" body = brotli.decompress(body) for key in ['Transfer-Encoding', 'Content-Encoding', 'Content-Length', 'Set-Cookie']: if key in headers: del headers[key] headers['Content-Length'] = len(body) headers = [f'{key}: {value}' for key, value in headers.items()] if setcookies is not None: setcookies = setcookies.split(",") ind = 0 while ind < len(setcookies): if setcookies[ind].lower()[:-4].endswith('expires'): headers.append(f"Set-Cookie: {setcookies[ind]}, {setcookies[ind + 1]}") ind += 2 else: headers.append(f"Set-Cookie: {setcookies[ind]}") ind += 1 headers = END_ONE_HEADER.join(headers) return (start_line + END_ONE_HEADER + headers + END_HEADERS).encode('ISO-8859-1') + body
def rewrite_request(req): try: while True: parser = HttpStream(req) headers = parser.headers() parsed_url = urlparse.urlparse(parser.url()) is_ssl = parsed_url.scheme == "https" host = get_host(parse_address(parsed_url.netloc, 80), is_ssl=is_ssl) headers['Host'] = host headers['Connection'] = 'close' if 'Proxy-Connection' in headers: del headers['Proxy-Connection'] location = urlparse.urlunparse( ('', '', parsed_url.path, parsed_url.params, parsed_url.query, parsed_url.fragment)) httpver = "HTTP/%s" % ".".join(map(str, parser.version())) new_headers = [ "%s %s %s\r\n" % (parser.method(), location, httpver) ] new_headers.extend(["%s: %s\r\n" % (hname, hvalue) \ for hname, hvalue in headers.items()]) req.writeall(bytes("".join(new_headers) + "\r\n")) body = parser.body_file() send_body(req, body, parser.is_chunked()) except (socket.error, NoMoreData, ParserError): pass
def rewrite_request(req): try: while True: parser = HttpStream(req) headers = parser.headers() parsed_url = urlparse.urlparse(parser.url()) is_ssl = parsed_url.scheme == "https" host = get_host(parse_address(parsed_url.netloc, 80), is_ssl=is_ssl) headers['Host'] = host headers['Connection'] = 'close' if 'Proxy-Connection' in headers: del headers['Proxy-Connection'] location = urlparse.urlunparse(('', '', parsed_url.path, parsed_url.params, parsed_url.query, parsed_url.fragment)) httpver = "HTTP/%s" % ".".join(map(str, parser.version())) new_headers = ["%s %s %s\r\n" % (parser.method(), location, httpver)] new_headers.extend(["%s: %s\r\n" % (hname, hvalue) \ for hname, hvalue in headers.items()]) req.writeall(bytes("".join(new_headers) + "\r\n")) body = parser.body_file() send_body(req, body, parser.is_chunked()) except (socket.error, NoMoreData, ParserError): pass
def send_downstream(self, p, rhost, rport, method, url, upsock): with self.connect_downstream(rhost, rport) as downsock: request = '{method} {url} HTTP/{version[0]}.{version[1]}\r\n{headers}\r\n\r\n'.format(method=method.decode(), url=url, version=p.version(), headers='\r\n'.join(['{0}: {1}'.format(name, value) for name, value in p.headers().items()])) self.logger.debug('sending request {0!r}'.format(request)) downsock.send(request.encode()) downstream = HttpStream(SocketReader(downsock)) self.logger.debug('response: header={0}'.format(downstream.headers())) upsock.send('HTTP/{version[0]}.{version[1]} {code} {status}\r\n{headers}\r\n\r\n'.format(version=downstream.version(), code=downstream.status_code(), status=downstream.status(), headers='\r\n'.join(['{0}: {1}'.format(name, value) for name, value in downstream.headers().items()])).encode()) upsock.send(downstream.body_string())
def handle(self, upsock, peer): with closing(upsock) as upsock: origaddr = upsock.getsockname() self.logger.debug('intercepted connection from {0} to {1}'.format(peer, origaddr)) p = HttpStream(SocketReader(upsock)) self.logger.debug('request: method={0} url={1} version={2} headers={3}'.format(p.method(), p.url(), p.version(), p.headers())) url = None if p.method() == b'CONNECT': self.handle_connect(p, upsock) elif p.url()[0] == '/': self.handle_transparent(p, upsock) else: self.handle_proxy(p, upsock)