def recv(self, wsgi=False):
try:
stream = HttpStream(self.reader, kind=HTTP_REQUEST, parser_class=HttpParser, decompress=True)
if bool(wsgi):
environ = stream.wsgi_environ()
environ['wsgi.url_scheme'] = guess_scheme(environ)
environ['wsgi.input'] = stream.body_file()
environ['wsgi.socket'] = self.socket
return environ
# BUG:
# http-parser has an issue here, if we call 'method' before 'headers'
# and invalid method name is returned...
fields = stream.headers()
method = stream.method()
url = stream.url()
version = stream.version()
content = stream.body_file()
url = urlparse(url)
path = url.path
query = parse_qs(url.query, keep_blank_values=True)
fragment = url.fragment
for k, v in six.iteritems(dict(query)):
query[k] = parse_query_value(v)
self.version = 'HTTP/%s.%s' % version
return method, path, query, fragment, fields, content
except NoMoreData:
pass
def get_response(self, request, connection):
""" return final respons, it is only accessible via peform
method """
if log.isEnabledFor(logging.DEBUG):
log.debug("Start to parse response")
p = HttpStream(SocketReader(connection.socket()), kind=1,
decompress=self.decompress)
if log.isEnabledFor(logging.DEBUG):
log.debug("Got response: %s %s" % (p.version(), p.status()))
log.debug("headers: [%s]" % p.headers())
location = p.headers().get('location')
if self.follow_redirect:
should_close = not p.should_keep_alive()
if p.status_code() in (301, 302, 307,):
# read full body and release the connection
p.body_file().read()
connection.release(should_close)
if request.method in ('GET', 'HEAD',) or \
self.force_follow_redirect:
if hasattr(self.body, 'read'):
try:
self.body.seek(0)
except AttributeError:
raise RequestError("Can't redirect %s to %s "
"because body has already been read"
% (self.url, location))
return self.redirect(location, request)
elif p.status_code() == 303 and self.method == "POST":
# read full body and release the connection
p.body_file().read()
connection.release(should_close)
request.method = "GET"
request.body = None
return self.redirect(location, request)
# create response object
resp = self.response_class(connection, request, p)
# apply response filters
for f in self.response_filters:
f.on_response(resp, request)
if log.isEnabledFor(logging.DEBUG):
log.debug("return response class")
# return final response
return resp
def get_response(self, request, connection):
""" return final respons, it is only accessible via peform
method """
if log.isEnabledFor(logging.DEBUG):
log.debug("Start to parse response")
p = HttpStream(SocketReader(connection.socket()), kind=1,
decompress=self.decompress)
if log.isEnabledFor(logging.DEBUG):
log.debug("Got response: %s %s" % (p.version(), p.status()))
log.debug("headers: [%s]" % p.headers())
location = p.headers().get('location')
if self.follow_redirect:
should_close = not p.should_keep_alive()
if p.status_code() in (301, 302, 307,):
# read full body and release the connection
p.body_file().read()
connection.release(should_close)
if request.method in ('GET', 'HEAD',) or \
self.force_follow_redirect:
if hasattr(self.body, 'read'):
try:
self.body.seek(0)
except AttributeError:
raise RequestError("Can't redirect %s to %s "
"because body has already been read"
% (self.url, location))
return self.redirect(location, request)
elif p.status_code() == 303 and self.method == "POST":
# read full body and release the connection
p.body_file().read()
connection.release(should_close)
request.method = "GET"
request.body = None
return self.redirect(location, request)
# create response object
resp = self.response_class(connection, request, p)
# apply response filters
for f in self.response_filters:
f.on_response(resp, request)
if log.isEnabledFor(logging.DEBUG):
log.debug("return response class")
# return final response
return resp
def recv(self):
try:
stream = HttpStream(self.reader, kind=HTTP_RESPONSE, parser_class=HttpParser, decompress=True)
status = stream.status_code()
version = stream.version()
fields = stream.headers()
content = stream.body_file()
self.version = 'HTTP/%s.%s' % version
return status, fields, content
except NoMoreData:
pass
def join_data(request: HttpStream, type_: str):
assert type_ in ('Request', 'Response')
# try:
major, minor = request.version()
# except ParserError:
# raise
if type_ == 'Request':
url = request.url()
start_line = f'{request.method()} {url} HTTP/{major}.{minor}'
else:
start_line = f'HTTP/{major}.{minor} {request.status()}'
body = request.body_string(binary=True)
headers = request.headers()
setcookies = headers['Set-Cookie'] if 'Set-Cookie' in headers else None
if "Content-Encoding" in headers:
assert headers["Content-Encoding"] == "br"
body = brotli.decompress(body)
for key in ['Transfer-Encoding', 'Content-Encoding', 'Content-Length', 'Set-Cookie']:
if key in headers:
del headers[key]
headers['Content-Length'] = len(body)
headers = [f'{key}: {value}' for key, value in headers.items()]
if setcookies is not None:
setcookies = setcookies.split(",")
ind = 0
while ind < len(setcookies):
if setcookies[ind].lower()[:-4].endswith('expires'):
headers.append(f"Set-Cookie: {setcookies[ind]}, {setcookies[ind + 1]}")
ind += 2
else:
headers.append(f"Set-Cookie: {setcookies[ind]}")
ind += 1
headers = END_ONE_HEADER.join(headers)
return (start_line + END_ONE_HEADER + headers + END_HEADERS).encode('ISO-8859-1') + body
def rewrite_request(req):
try:
while True:
parser = HttpStream(req)
headers = parser.headers()
parsed_url = urlparse.urlparse(parser.url())
is_ssl = parsed_url.scheme == "https"
host = get_host(parse_address(parsed_url.netloc, 80),
is_ssl=is_ssl)
headers['Host'] = host
headers['Connection'] = 'close'
if 'Proxy-Connection' in headers:
del headers['Proxy-Connection']
location = urlparse.urlunparse(
('', '', parsed_url.path, parsed_url.params, parsed_url.query,
parsed_url.fragment))
httpver = "HTTP/%s" % ".".join(map(str, parser.version()))
new_headers = [
"%s %s %s\r\n" % (parser.method(), location, httpver)
]
new_headers.extend(["%s: %s\r\n" % (hname, hvalue) \
for hname, hvalue in headers.items()])
req.writeall(bytes("".join(new_headers) + "\r\n"))
body = parser.body_file()
send_body(req, body, parser.is_chunked())
except (socket.error, NoMoreData, ParserError):
pass
def rewrite_request(req):
try:
while True:
parser = HttpStream(req)
headers = parser.headers()
parsed_url = urlparse.urlparse(parser.url())
is_ssl = parsed_url.scheme == "https"
host = get_host(parse_address(parsed_url.netloc, 80),
is_ssl=is_ssl)
headers['Host'] = host
headers['Connection'] = 'close'
if 'Proxy-Connection' in headers:
del headers['Proxy-Connection']
location = urlparse.urlunparse(('', '', parsed_url.path,
parsed_url.params, parsed_url.query, parsed_url.fragment))
httpver = "HTTP/%s" % ".".join(map(str,
parser.version()))
new_headers = ["%s %s %s\r\n" % (parser.method(), location,
httpver)]
new_headers.extend(["%s: %s\r\n" % (hname, hvalue) \
for hname, hvalue in headers.items()])
req.writeall(bytes("".join(new_headers) + "\r\n"))
body = parser.body_file()
send_body(req, body, parser.is_chunked())
except (socket.error, NoMoreData, ParserError):
pass
def send_downstream(self, p, rhost, rport, method, url, upsock):
with self.connect_downstream(rhost, rport) as downsock:
request = '{method} {url} HTTP/{version[0]}.{version[1]}\r\n{headers}\r\n\r\n'.format(method=method.decode(), url=url, version=p.version(), headers='\r\n'.join(['{0}: {1}'.format(name, value) for name, value in p.headers().items()]))
self.logger.debug('sending request {0!r}'.format(request))
downsock.send(request.encode())
downstream = HttpStream(SocketReader(downsock))
self.logger.debug('response: header={0}'.format(downstream.headers()))
upsock.send('HTTP/{version[0]}.{version[1]} {code} {status}\r\n{headers}\r\n\r\n'.format(version=downstream.version(), code=downstream.status_code(), status=downstream.status(), headers='\r\n'.join(['{0}: {1}'.format(name, value) for name, value in downstream.headers().items()])).encode())
upsock.send(downstream.body_string())
def handle(self, upsock, peer):
with closing(upsock) as upsock:
origaddr = upsock.getsockname()
self.logger.debug('intercepted connection from {0} to {1}'.format(peer, origaddr))
p = HttpStream(SocketReader(upsock))
self.logger.debug('request: method={0} url={1} version={2} headers={3}'.format(p.method(), p.url(), p.version(), p.headers()))
url = None
if p.method() == b'CONNECT':
self.handle_connect(p, upsock)
elif p.url()[0] == '/':
self.handle_transparent(p, upsock)
else:
self.handle_proxy(p, upsock)
