def from_headers(strheader):
"""Parse cookie data from a string in HTTP header (RFC 2616) format.
@return: list of cookies
@raises: ValueError for incomplete or invalid data
"""
res = []
fp = StringIO(strheader)
headers = HTTPMessage(fp, seekable=True)
if "Host" not in headers:
raise ValueError("Required header 'Host:' missing")
host = headers["Host"]
# XXX: our --help says we also pay attention to the Scheme: header,
# but we don't?!
path = headers.get("Path", "/")
for header in headers.getallmatchingheaders("Set-Cookie"):
headervalue = header.split(':', 1)[1]
for pairs in split_header_words([headervalue]):
for name, value in pairs:
cookie = requests.cookies.create_cookie(name,
value,
domain=host,
path=path)
res.append(cookie)
return res
def handle_read(self, conn):
m = HTTPMessage()
status = 200
try:
msg = conn.readAll()
request = HTTPRequest(bytes(msg))
if request.path.startswith('/command/'):
command = request.path.split('/')[2]
args = request.path.split('/')[3:]
if command not in short_commands:
status = 405
m.set_payload('command not found').as_string()
ret = short_commands[command](self.controller, args)
m.set_payload(ret)
else:
status = 405
m.set_payload('invalid route')
except Exception as exc:
#TODO: 500 error
status = 500
m.set_payload('error!')
raise exc
finally:
conn.write('HTTP/1.1 %d %s\n%s' %
(status, responses[status], m.as_string()))
conn.disconnectFromHost()
def test_normalize_headers_message_dupes():
headers = HTTPMessage()
headers.add_header('Cookie', 'foo')
headers.add_header('Cookie', 'bar')
new = normalize_headers(headers)
assert len(new) == 1
assert new['cookie'] == 'foo, bar'
def _prepare_outgoing_headers(headers):
if headers is None:
headers = HTTPMessage()
elif not isinstance(headers, HTTPMessage):
new_headers = HTTPMessage()
if hasattr(headers, "items"):
iterator = headers.items()
else:
iterator = iter(headers)
for k, v in iterator:
new_headers[k] = v
headers = new_headers
_setdefault_header(headers, "User-Agent", DEFAULT_UA)
return headers
def assert_header_parsing(headers: httplib.HTTPMessage) -> None:
"""
Asserts whether all headers have been successfully parsed.
Extracts encountered errors from the result of parsing headers.
Only works on Python 3.
:param http.client.HTTPMessage headers: Headers to verify.
:raises urllib3.exceptions.HeaderParsingError:
If parsing errors are found.
"""
# This will fail silently if we pass in the wrong kind of parameter.
# To make debugging easier add an explicit check.
if not isinstance(headers, httplib.HTTPMessage):
raise TypeError(f"expected httplib.Message, got {type(headers)}.")
unparsed_data = None
# get_payload is actually email.message.Message.get_payload;
# we're only interested in the result if it's not a multipart message
if not headers.is_multipart():
payload = headers.get_payload()
if isinstance(payload, (bytes, str)):
unparsed_data = payload
# httplib is assuming a response body is available
# when parsing headers even when httplib only sends
# header data to parse_headers() This results in
# defects on multipart responses in particular.
# See: https://github.com/urllib3/urllib3/issues/800
# So we ignore the following defects:
# - StartBoundaryNotFoundDefect:
# The claimed start boundary was never found.
# - MultipartInvariantViolationDefect:
# A message claimed to be a multipart but no subparts were found.
defects = [
defect
for defect in headers.defects
if not isinstance(
defect, (StartBoundaryNotFoundDefect, MultipartInvariantViolationDefect)
)
]
if defects or unparsed_data:
raise HeaderParsingError(defects=defects, unparsed_data=unparsed_data)
def _build_transport_with_reply(self, body, status=200, pattern=None, test_request=None):
headers = HTTPMessage()
headers.add_header('Content-Type', 'text/xml; charset=utf-8')
reply = Reply(status, headers, body)
transport = HttpTransport()
def surrogate(request, *args, **kwargs):
if pattern and not re.search(pattern, request.url):
return HttpTransport.send(transport, *args, **kwargs)
if test_request:
test_request(request)
return reply
transport.send = mock.MagicMock()
transport.send.side_effect = surrogate
return transport
def extract_cookies_to_jar(jar, request, response):
"""Extract the cookies from the response into a CookieJar.
:param jar: cookielib.CookieJar (not necessarily a RequestsCookieJar)
:param request: our own requests.Request object
:param response: httpcore.Response object
"""
msg = HTTPMessage()
for k, v in response.headers.raw:
msg.add_header(k.decode(), v.decode())
# the _original_response field is the wrapped httplib.HTTPResponse object,
req = requests.cookies.MockRequest(request)
# pull out the HTTPMessage with the headers and put it in the mock:
res = requests.cookies.MockResponse(msg)
jar.extract_cookies(res, req)
def get_content_type_from_headers(res_headers: HTTPMessage):
if not res_headers:
return None
content_type: str = res_headers.get('Content-Type', failobj=None)
return get_content_type_from_header(content_type)
def __init__(self, status):
super(MockHttpResponse, self).__init__()
self.status = status
self.strict = 0
self.version = 0
self.reason = None
self.msg = HTTPMessage(io.BytesIO())
def parse_headers(data):
if sys.version_info > (3, 0):
return email.parser.Parser(_class=HTTPMessage).parsestr(
data.decode('iso-8859-1'))
else:
fp = io.StringIO(data.decode('iso-8859-1'))
return HTTPMessage(fp, 0)
def test_generate_handshake_response_raise_key_error(
self,
http_request: HTTPRequest,
) -> None:
http_request.headers = HTTPMessage()
with pytest.raises(InvalidSecWebsocketKey):
generate_handshake_response(http_request)
def _dirport_mock(data, encoding = 'identity'):
dirport_mock = Mock()
dirport_mock().read.return_value = data
if stem.prereq.is_python_3():
headers = HTTPMessage()
for line in HEADER.splitlines():
key, value = line.split(': ', 1)
headers.add_header(key, encoding if key == 'Content-Encoding' else value)
dirport_mock().headers = headers
else:
dirport_mock().headers = HTTPMessage(io.BytesIO(HEADER % encoding))
return dirport_mock
def _prepare_incoming_headers(headers):
headers_dict = {}
for k, v in headers.items():
headers_dict.setdefault(k, []).append(v)
result = HTTPMessage()
# note that iterating over headers_dict preserves the original
# insertion order in all versions since Python 3.6:
for k, vlist in headers_dict.items():
result[k] = ",".join(vlist)
return result
def _make_response(self, result, url):
data = '\r\n'.join([ '%s: %s' % (k, v) for k, v in result.header_items ])
if PY2:
headers = HTTPMessage(BytesIO(data))
else:
import email
headers = email.message_from_string(data)
response = addinfourl(BytesIO(result.data), headers, url)
code, msg = result.status.split(None, 1)
response.code, response.msg = int(code), msg
return response
def __init__(self, connection, address, conn_id):
"""
Create a ConnectionHandler class to handle the incoming HTTP(S) request.
:param connection: Socket belonging to the incoming connection.
:type connection: socket
:param address: Address of the connecting party.
:type address: host, port
"""
self.conn_id = conn_id
self.connection = connection
self.method = self.path = self.protocol = None
self.headers = HTTPMessage()
cur_thread = threading.current_thread()
cur_thread.name = self.conn_id
try:
if not self.parse_request():
# FIXME(kormat): need error reporting
return
self.handle_request()
finally:
cleanup(self.connection)
def mock_httpresponse(code=200, content_type="/xml"):
"""Create a mocked version of a response object returned by urlopen().
:param int code: Value of "code" attribute.
:param str content_type: Used to set the "Content-Type" header in the "headers" attribute. This
is checked in Entrez._open() to determine if the response data is plain text.
"""
resp = mock.NonCallableMock()
resp.code = code
resp.headers = HTTPMessage()
resp.headers.add_header("Content-Type", content_type + "; charset=UTF-8")
return resp
def _create_mock_resonse(self):
mock_response = Mock()
mock_response.status = 200
mock_response.reason = 'OK'
headers = HTTPMessage()
headers.add_header('Content-Type', 'application/json')
headers.add_header('Content-Length', '500')
mock_response.headers = headers
return mock_response
def _create_mock_request(self, path='http://www.example.com/test/path/'):
mock_request = Mock()
mock_request.path = path
mock_request.command = 'GET'
headers = HTTPMessage()
headers.add_header('Host', 'www.example.com')
headers.add_header('Accept', '*/*')
mock_request.headers = headers
return mock_request
def handle_request(self, req, req_body):
"""Captures a request and its body.
Args:
req: The request (an instance of CaptureRequestHandler).
req_body: The binary request body.
"""
# First make any modifications to the request
# DEPRECATED. This will be replaced by request_interceptor
req.body = req_body # Temporarily attach the body to the request for modification
self.server.modifier.modify_request(req,
urlattr='path',
methodattr='command')
req_body = req.body
# Convert the implementation specific request to one of our requests
# for handling.
request = self._create_request(req, req_body)
# Call the request interceptor if set
if self.server.request_interceptor is not None:
self.server.request_interceptor(request)
if request.response:
# The interceptor has created a response for us to send back immediately
self.commit_response(request.response.status_code,
request.response.reason,
request.response.headers,
request.response.body)
return False # Signals that we've committed the response ourselves
# Transfer any modifications to the original request
req.command = request.method
req.path = request.url
req.headers = HTTPMessage()
for name, val in request.headers.items():
req.headers.add_header(name, val)
if request.body:
req_body = request.body
self.capture_request(request)
if request.id is not None: # Will not be None when captured
req.id = request.id
return req_body
def mock_urlopen(request, *_, **__):
""" Create a mock HTTP response. """
# request has data, header, method
data = dumps({
"data": request.data.decode() if request.data else None,
"query": urlparse(request.full_url).query
}).encode()
fields = (
("Content-Type", "application/json"),
("Last-Modified", "Thu, 01 Jan 1970 00:00:00 GMT"),
("Set-Cookie", "name=cookie1"),
("Set-Cookie", "name=cookie2"),
)
headers = HTTPMessage()
for key, value in fields:
headers[key] = value
return _MockResponse(data, headers)
def handle_response(self, req, req_body, res, res_body):
"""Captures a response and its body that relate to a previous request.
Args:
req: The original request (an instance of CaptureRequestHandler).
req_body: The body of the original request.
res: The response (a http.client.HTTPResponse instance) that corresponds to the request.
res_body: The binary response body.
"""
# Make any modifications to the response
# DEPRECATED. This will be replaced by response_interceptor.
self.server.modifier.modify_response(res, req, urlattr='path')
if not hasattr(req, 'id'):
# Request was not captured
return
# Convert the implementation specific response to one of our responses
# for handling.
response = Response(status_code=res.status,
reason=res.reason,
headers=res.headers.items(),
body=res_body)
# Call the response interceptor if set
if self.server.response_interceptor is not None:
self.server.response_interceptor(
self._create_request(req, req_body, response), response)
# Transfer any modifications to the original response
res.status = response.status_code
res.reason = response.reason
res.headers = HTTPMessage()
for name, val in response.headers.items():
res.headers.add_header(name, val)
if response.body:
res_body = response.body
self.capture_response(req.id, req.path, response)
return res_body
def test_detect_charset(self):
"""
Test ConnectorMixin.detect_charset() method
"""
# Test 1
# Encoding data is in the response header, get_content_charset() returns an encoding.
headers = HTTPMessage()
headers.get_content_charset = MagicMock(return_value='euc-kr')
charset = connectors.ConnectorMixin.detect_charset(headers, b'')
self.assertEqual('euc-kr', charset)
# Test 2
# Sometimes response does not have 'Content-Type' header.
# Find meta tag in the content: <meta charset="...">
headers.get_content_charset = MagicMock(return_value='')
content = b'<html><head><meta charset="utf-16"></head></html>'
charset = connectors.ConnectorMixin.detect_charset(headers, content)
self.assertEqual('utf-16', charset)
# Test 3
# <meta http-equiv ... > style tag
headers.get_content_charset = MagicMock(return_value='')
content = b"""
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=test-encoding">
</head>
</html>
"""
charset = connectors.ConnectorMixin.detect_charset(headers, content)
self.assertEqual('test-encoding', charset)
# Test 4
# Test if fallback encoding is correctly returned
headers.get_content_charset = MagicMock(return_value='')
content = b"""<html><head></head></html>"""
charset = connectors.ConnectorMixin.detect_charset(headers, content, 'fallback-encoding')
self.assertEqual('fallback-encoding', charset)
def instance():
return Headers(HTTPMessage())
def info(self):
message = HTTPMessage()
message.headers = (('Content-type',
'text/plain; charset=ISO-8859-1'), )
return message
def __init__(self, set_cookie_headers):
msg = HTTPMessage()
for set_cookie_header in set_cookie_headers:
msg.add_header("Set-Cookie", set_cookie_header)
self._info = msg
def http_request(self) -> HTTPRequest:
headers = HTTPMessage()
headers.add_header("Sec-WebSocket-Key", self.client_key)
_request = HTTPRequest()
_request.headers = headers
return _request
def info(self):
message = HTTPMessage()
message.headers = (
('Content-type', 'text/plain; charset=ISO-8859-1'),
)
return message
class ConnectionHandler(object):
"""
Handler class for the connection to be proxied.
"""
server_version = "SCION HTTP Proxy/" + VERSION
def __init__(self, connection, address, conn_id):
"""
Create a ConnectionHandler class to handle the incoming HTTP(S) request.
:param connection: Socket belonging to the incoming connection.
:type connection: socket
:param address: Address of the connecting party.
:type address: host, port
"""
self.conn_id = conn_id
self.connection = connection
self.method = self.path = self.protocol = None
self.headers = HTTPMessage()
cur_thread = threading.current_thread()
cur_thread.name = self.conn_id
try:
if not self.parse_request():
# FIXME(kormat): need error reporting
return
self.handle_request()
finally:
cleanup(self.connection)
def parse_request(self):
"""
Extracts the request line of an incoming HTTP(S) request.
:returns: HTTP(S) Method, Path, HTTP(S) Protocol Version
:rtype: triple
"""
data = []
lf_count = 0
while lf_count < 2:
b = self.connection.recv(1)
if not b:
logging.info("Client closed the connection.")
return False
if b == b"\r":
# Drop \r's, as recommended by rfc2616 19.3
continue
data.append(b)
if b == b"\n":
lf_count += 1
else:
lf_count = 0
lines = b"".join(data).decode("ascii").split("\n")
self.method, self.path, self.protocol = lines.pop(0).split(" ")
for line in lines:
if not line:
break
self.headers.add_header(*line.split(": ", 1))
logging.info("Request: %s %s %s", self.method, self.path,
self.protocol)
logging.debug("Request headers:\n%s", self.headers)
return True
def handle_request(self):
if self.method == 'CONNECT':
self.do_CONNECT()
elif self.method in ('GET', 'HEAD', 'POST', 'PUT', 'DELETE'):
self.handle_others()
else:
# FIXME(kormat): need error reporting
logging.warning("Invalid HTTP(S) request")
def do_CONNECT(self):
"""
Handles the CONNECT method: Connects to the target address,
and responds to the client (i.e. browser) with a 200
Connection established response and starts proxying.
"""
soc = self._connect_to(self.path)
if not soc:
# FIXME(kormat): needs error handling
return
reply = "\r\n".join([
"HTTP/1.1 200 Connection established",
"Proxy-agent: %s" % self.server_version
]) + "\r\n\r\n"
try:
self.connection.send(reply.encode("ascii"))
self._read_write(soc)
finally:
cleanup(soc)
def handle_others(self):
"""
Handles the rest of the supported HTTP methods: Parses the path,
connects to the target address, sends the complete request
to the target and starts proxying.
"""
(scm, netloc, path, params, query, _) = urlparse(self.path, 'http')
if scm != 'http' or not netloc:
logging.error("Bad URL %s" % self.path)
return
conn_hdr = self.headers["Connection"]
if conn_hdr:
del self.headers[conn_hdr]
self.headers.replace_header("Connection", "close")
soc = self._connect_to(netloc)
if not soc:
# FIXME(kormat): needs error handling
return
try:
# Normal mode proxy strips out scm and netloc
self._send_request(soc, '', '', path, params, query)
self._read_write(soc)
finally:
cleanup(soc)
logging.debug("Done")
def _connect_to(self, netloc):
"""
Establishes a connection to the target host.
:param netloc: The hostname (and port) of the target to be connected to.
:type netloc: string
:returns: The socket that is used to connect.
:rtype: socket
"""
soc = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
if ':' in netloc:
host, port = netloc.split(':')
else:
host, port = netloc, 80
logging.debug("Connecting to %s:%s" % (host, port))
try:
soc.connect((host, int(port)))
except OSError:
log_exception("Error while connecting to %s:%s" % (host, port))
return False
logging.debug("Connected to %s:%s" % (host, port))
return soc
def _send_request(self, soc, scm, netloc, path, params, query):
"""
Helper function that prepares and sends the request on the
given socket.
:param soc: The socket the request is going to be sent on.
:type soc: socket
:param path: The path of the HTTP request.
:type path: String
:param params: Parameters of the request (if any).
:type params: String
:param query: Query section of the HTTP request (if any).
:type query: String
"""
base = "%s %s HTTP/1.0" % (self.method,
urlunparse(
(scm, netloc, path, params, query, '')))
req = []
req.append(base)
for hdr, val in self.headers.items():
req.append("%s: %s" % (hdr, val))
req_bytes = ("\r\n".join(req) + "\r\n\r\n").encode("ascii")
logging.debug("Sending a request: %s", req_bytes)
# FIXME(kormat): need error handling/reporting
soc.send(req_bytes)
def _read_write(self, target_sock):
"""
The main function responsible for the proxying operation. It creates
two threads to listen for incoming data on both client (i.e. browser)
and server sockets and relays them accordingly between each other.
:param target_sock: The socket belonging to the remote target.
:type target_sock: socket
"""
t1 = threading.Thread(target=thread_safety_net,
args=(ProxyData, self.connection, target_sock),
name="%s-c2s" % self.conn_id)
t1.start()
t2 = threading.Thread(target=thread_safety_net,
args=(ProxyData, target_sock, self.connection),
name="%s-s2c" % self.conn_id)
t2.start()
# Wait until both threads finish.
t1.join()
t2.join()
def handle_read(self, conn):
m = HTTPMessage()
status = 200
try:
msg = conn.readAll()
request = HTTPRequest(bytes(msg))
if request.path.startswith('/command/'):
command = request.path.split('/')[2]
args = request.path.split('/')[3:]
if command not in short_commands:
status = 405
m.set_payload('command not found').as_string()
ret = short_commands[command](self.controller, args)
m.set_payload(ret)
else:
status = 405
m.set_payload('invalid route')
except Exception as exc:
#TODO: 500 error
status = 500
m.set_payload('error!')
raise exc
finally:
conn.write('HTTP/1.1 %d %s\n%s' %
(status, responses[status], m.as_string()))
conn.disconnectFromHost()
