def test_chunked_w_trailer() -> str:
client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
client.connect((config.SERVER_ADDR, config.SERVER_PORT))
request_header = "POST /post/print.cgi HTTP/1.1\r\nHost: localhost\r\nTransfer-encoding: chunked\r\n\r\n"
client.send(request_header.encode())
request_header = "5\r\ntest\n\r\n"
client.send(request_header.encode())
request_header = "0\r\naccept-language: fr\r\ntest-header: blabla\r\n\r\n"
client.send(request_header.encode())
# read and parse http response
http_response = HTTPResponse(client)
http_response.begin()
if http_response.status != 226:
return "Bad status code: {}, expected: {}".format(
str(http_response.status), "226")
if http_response.headers["Content-Type"] != "CGI/MINE":
return "Bad content-type: {}, expected: {}".format(
http_response.headers["Content-Type"], "CGI/MINE")
body = http_response.read().decode("UTF-8")
# print(body)
if (body.find("HTTP_TEST_HEADER=blabla") == -1
or body.find("HTTP_ACCEPT_LANGUAGE=fr") == -1):
return "Missing headers from request"
if body.find("test") == -1:
return "Missing content in request"
# print(body)
return ""
def urllib3_response_from_bytes(data: bytes) -> HTTPResponse:
sock = BytesIOSocket(data)
response = HTTPResponse(sock)
response.begin()
return urllib3.HTTPResponse.from_httplib(response)
def _parse_http_response(self, response_str):
"""Parse response string into an HTTP object."""
sock = TmpSock(response_str)
res = HTTPResponse(sock)
res.begin()
print('res', res.getheaders())
return res
def success(self, data: HTTPResponse) -> dict:
return {
'code': data.status,
'reason': data.reason,
'headers': data.getheaders(),
'data': data.read()
}
def discover(timeout=2, retries=1, st=ST_ECP):
group = ('239.255.255.250', 1900)
message = '\r\n'.join([
'M-SEARCH * HTTP/1.1',
'HOST: {0}:{1}'.format(*group),
'MAN: "ssdp:discover"',
'ST: {st}', 'MX: 3', '', ''])
socket.setdefaulttimeout(timeout)
responses = {}
for _ in range(retries):
sock = socket.socket(
socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, 2)
m = message.format(st=st)
sock.sendto(m.encode(), group)
while 1:
try:
rhttp = HTTPResponse(_FakeSocket(sock.recv(1024)))
rhttp.begin()
if rhttp.status == 200:
rssdp = SSDPResponse(rhttp)
responses[rssdp.location] = rssdp
except socket.timeout:
break
return responses.values()
def parse_http_response(buf):
"""Get parsed HTTP response from request buffer."""
src = FakeSocket(buf)
resp = HTTPResponse(src)
resp.begin()
return resp
def __init__(self, sock, debuglevel=0, method=None, **kwargs):
# We have to use a positive debuglevel to get it passed to here,
# however we don't want to use it because by default debugging prints
# to the stdout and we can't capture it, so we use a special -1 value
if debuglevel == 5:
debuglevel = -1
HTTPResponse.__init__(self, sock, debuglevel=debuglevel, method=method)
def processHttpResponse(http_response_text, key_enrich, value_enrich):
return_http_response = bytearray(b'')
response = None
http_first_chars = b''
if (len(http_response_text) > 4):
http_first_chars = http_response_text[:4]
if (http_first_chars == b'HTTP'):
response = HTTPResponse(HTTPResponseSocket(http_response_text))
response.begin()
##??? print (str(http_first_chars))
if (response != None):
str_version = "HTTP/1.0"
if (response.version == 11):
str_version = "HTTP/1.1"
line_text = ( str_version + ' ' +
str(response.status) + ' ' + str(response.reason) + '\r\n' )
return_http_response.extend(line_text.encode('utf-8'))
return_http_response.extend(
modify_http_header(response.getheaders(), key_enrich, value_enrich))
return_http_response.extend(response.read())
return return_http_response
def response_from_bytes(data):
sock = BytesIOSocket(data)
http_response = HTTPResponse(sock)
http_response.begin()
return urllib3.HTTPResponse.from_httplib(http_response)
def __init__(self, sock, debuglevel=0, method=None, **kwargs):
# We have to use a positive debuglevel to get it passed to here,
# however we don't want to use it because by default debugging prints
# to the stdout and we can't capture it, so we use a special -1 value
if debuglevel == 5:
debuglevel = -1
HTTPResponse.__init__(self, sock, debuglevel=debuglevel, method=method)
def inner(code, location):
r = HTTPResponse(
Sock(RESP % (code, codes[code].encode(), location.encode())),
method='GET',
url='https://foo.bar/',
)
r.begin()
return r
def _decode_json_response(r:HTTPResponse) -> Any:
""" Decode JSON HTTP response """
charset = r.getheader('charset', 'utf-8')
if r.headers.get_content_type() != 'application/json':
return None
return json.loads(r.read().decode(charset))
def get_response_object(data) -> Union[HTTPResponse, None]:
try:
source = FakeSocket(data)
response = HTTPResponse(source)
response.begin()
return response
except HTTPException:
return None
def send_request(request_header: str) -> str:
client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
client.connect((config.SERVER_ADDR, config.SERVER_PORT))
client.send(request_header.encode())
# read and parse http response
http_response = HTTPResponse(client)
http_response.begin()
return http_response
def evaluate_response(response: HTTPResponse) -> str:
if response.code() == 404:
return "Not Found"
elif response.code() == 502:
return "???"
elif response.code() == 400:
return "???"
else:
return "Unknown Status Code"
def evaluate_response(response: HTTPResponse) -> str:
if response.code() == HTTPCode.NOT_FOUND:
return "Not Found"
elif response.code() == HTTPCode.BAD_GATEWAY:
return "???"
elif response.code() == HTTPCode.BAD_REQUEST:
return "???"
else:
return "Unknown Status Code"
def _http_response_asserts(response: HTTPResponse, fuzz_data_logger):
if response.status >= 500:
fuzz_data_logger.log_fail("Status code higher or equal than 500!")
if response.getheader("Content-Type") == "application/json":
try:
json.loads(response.read())
except ValueError:
fuzz_data_logger.log_fail(
"application/json body is not valid JSON structure")
def __init__(self,
sock,
debuglevel=0,
strict=0,
method=None,
buffering=False):
HTTPResponse.__init__(self, sock, debuglevel, strict, method,
buffering)
self._refcount = 0
def getcontent(self):
"""Return content after parsing html
Returns:
response content that has attributes(.header and .content)
"""
response = HTTPResponse(self.sock)
response.begin()
return urllib3.HTTPResponse.from_httplib(response)
def parse_response(self, response_text):
"""
Given a raw HTTP response, create a httplib.HTTResponse out of it.
"""
source = FakeSocket(response_text)
response = HTTPResponse(source)
response.begin()
res = response.read()
return (res, response.status)
def response_length(resp: HTTPResponse):
"""In case of chunked encoding, calc length from content-range. TODO"""
headers = dict((key.lower(), val) for key, val in resp.getheaders())
if resp.getcode() == 206 and 'content-range' in headers:
match = re.match(r' *bytes *(\d+) *- *(\d+)', headers['content-range'])
if match is None:
raise RuntimeError("unexpected content-range: %s" % headers['content-range'])
start_byte, end_byte = match.groups()
return int(end_byte) - int(start_byte) + 1
else:
return resp.length
def _parse(read_method: Callable) -> HTTPResponse:
"""Trick to standardize the API between sockets and SSLConnection objects.
"""
response = read_method(4096)
while b'HTTP/' not in response or b'\r\n\r\n' not in response:
# Parse until the end of the headers
response += read_method(4096)
fake_sock = _FakeSocket(response)
response = HTTPResponse(fake_sock) # type: ignore
response.begin()
return response
def make_mock_response(status, text):
response_string = (
"HTTP/1.1 {0} Reason\r\n" "Content-Type: application/json\r\n" "\r\n" "{1}"
).format(status, text)
mocked_sock = MagicMock()
mocked_sock.makefile.return_value = BytesIO(response_string.encode())
http_response = HTTPResponse(mocked_sock)
http_response.begin()
return http_response
def from_http_client_response(obj: HTTPResponse) -> Response:
body = obj.read()
res = ResponseBuilder.from_dict(
dict(
statusCode=obj.getcode(),
headers={k: v
for k, v in obj.getheaders()},
body=body if isinstance(body, str) else
body.decode("utf8") if isinstance(body, bytes) else None,
))
ResponseBuilder.validate(res)
return res
def _parse(read_method: Callable) -> HTTPResponse:
"""Trick to standardize the API between sockets and SSLConnection objects.
"""
response = read_method(4096)
while b'HTTP/' not in response or b'\r\n\r\n' not in response:
# Parse until the end of the headers
response += read_method(4096)
fake_sock = _FakeSocket(response)
response = HTTPResponse(fake_sock) # type: ignore
response.begin()
return response
def get_body_str(res: HTTPResponse) -> str:
try:
for h in res.getheaders():
if h[0] == "Content-Type":
if "charset=" in h[1].split(';')[1]:
encoding: str = h[1].split(';')[1].split("=")[1]
except:
encoding: str = "utf-8"
result: str = ''
for line in res.readlines():
result += (line.decode(encoding))
return result
def parse_Response(res_file):
try:
response_str = res_file
source = FakeSocket(response_str.encode('utf-8'))
response = HTTPResponse(source)
response.begin()
# response_body = response.read()
except Exception as e:
return {
"error":
"Failed to Parse Given HTTP Raw Response. Please Verify your Input and Try Again."
}
return response
def test_ok_response_mime_type_is_there(fake_socket):
"""Test that request has a proper mime_type."""
from server import response_ok
if sys.version_info.major == 3:
from http.client import HTTPResponse
else:
from httplib import HTTPResponse
response_str = response_ok(b'htmlhtml', 'text/plain')
source = fake_socket(response_str)
response = HTTPResponse(source)
response.begin()
assert response.getheader('Content-Type') == 'text/plain'
def test_resolve_uri_to_response_ok_working_for_html(fake_socket):
"""Test that response_ok returns the file from resolve_uri as the body."""
from server import response_ok, resolve_uri
if sys.version_info.major == 3:
from http.client import HTTPResponse
else:
from httplib import HTTPResponse
response_str = response_ok(*resolve_uri('/a_web_page.html'))
source = fake_socket(response_str)
response = HTTPResponse(source)
response.begin()
assert response.read(len(response_str)) == b"""<!DOCTYPE html>
def test_ok_response_body_is_there(fake_socket):
"""Test that request has a body."""
from server import response_ok
if sys.version_info.major == 3:
from http.client import HTTPResponse
else:
from httplib import HTTPResponse
response_str = response_ok(b'htmlhtml', 'text/plain')
source = fake_socket(response_str)
response = HTTPResponse(source)
response.begin()
assert response.read(len(response_str)) == b'htmlhtml'
def myProxy(self):
if self.remote is None or self.lastHost != self.headers["Host"]:
self.remote = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.remote.settimeout(RemoteTimeout)
if ':' in self.headers["Host"]:
port = int(self.headers["Host"].split(':')[1])
else:
port = 80
self.remote.connect((self.headers["Host"].split(':')[0], port))
if not self.headers['Host'].startswith('http'):self.headers['Host'] = 'http://' + self.headers['Host']
if 'Cookie' in self.headers:
self.sessionid = self.headers['Cookie']
open('.session', 'w').write(self.headers['Cookie'])
# print sessionid
else:
sessionid = ''
if self.requestline.startswith('HEAD /'): # Adjust 'HEAD /xxx' situation
self.requestline = self.requestline.replace('HEAD ', 'HEAD ' + self.headers['Host'])
self.remote.sendall(self.requestline.encode('ascii') + b"\r\n")
headerstr = str(self.headers).replace("\r\n", "\n").replace("\n", "\r\n")
self.remote.sendall(headerstr.encode('ascii', 'ignore') + b"\r\n")
# Send Post data
if self.command == 'POST':
postdata = self.rfile.read(int(self.headers['Content-Length']))
self.remote.sendall(postdata)
# if 'cardselect/savedeckcard' in self.requestline:
# open('.carddeck', 'w').write(postdata)
response = HTTPResponse(self.remote, method = self.command)
response.begin()
# Reply to the browser
status = "HTTP/1.1 " + str(response.status) + " " + response.reason
self.wfile.write(status.encode('ascii') + b'\r\n')
hlist = []
for line in response.msg.headers: # Fixed multiple values of a same name
if 'TRANSFER-ENCODING' not in line.upper():
hlist.append(line)
self.wfile.write("".join(hlist) + b'\r\n')
if self.command == "CONNECT" and response.status == 200:
return self.transfer(self.remote, self.connection)
elif self.command == "HEAD":
pass
else:
r = ''
while True:
response_data = response.read(BufferSize)
if not response_data: break
self.wfile.write(response_data)
def process_file(filename):
global file_cntr, resp_file_cntr, inspected_cntr, valid_cntr
with open(filename, 'rb') as file:
file_cntr += 1 #increment examined file counter
data = file.read(100000000) #100MB
#data = file.read()
#there may be more than one response in one stream so bytes are splitten in
#place of (for example) HTTP/1.1 200 which should indicate response start
splitlist = re.split(b'(HTTP/\d.\d \d{3})', data)
splitlistlen = len(splitlist) #length of split list
#print(splitlist)
#length have to be odd:
# first item should be empty (bytes before first header)
# second item is header (HTTP/1.1 200)
# third item is rest of response
# same for next responses
if not (splitlistlen % 2):
sys.stderr.write('Splitlist is even in {}\n'.format(filename))
exit(1)
if splitlist[0] == data:
#there is no HTTP request found at all
return
if splitlist[0] != b'':
#there are some data before first response
#sys.stderr.write('Data before first request in {}\n'.format(filename))
pass
resp_file_cntr += 1
#for every response header
for i in range(1, len(splitlist), 2):
inspected_cntr += 1 #increment counter
#join header with rest of the response, create fake socket from
#response bytes and pass it into HTTPResponse constructor
resp = HTTPResponse(FakeSocket(splitlist[i] + splitlist[i + 1]))
payload = bytes()
try:
resp.begin() #HTTPResponse object have to be initialized
payload = resp.read()
except:
continue
##################################################
#now the response is parsed and ready
valid_cntr += 1
#count_headers(resp.getheaders())
#content_type(resp.getheaders())
if payload != b'':
process_payload(resp.getheaders(), payload)
def get_content(response: HTTPResponse) -> str:
"""Get content from HTTP response.
Handles gzipped content.
:param HTTPResponse response: HTTP response
:returns: HTTP response content
:rtype: str
"""
content = response.read()
if response.getheader('Content-encoding') == 'gzip':
content = gzip.decompress(content)
return content.decode()
def close(self): # Consume any remaining data in the socket try: while select((self.fp._sock,), (), (), 0)[0]: if not self.fp._sock.recv(256): break except AttributeError: pass HTTPResponse.close(self) try: self._handler.free_connection(self._connection) except AttributeError: pass
def __init__(self, response):
r = HTTPResponse(self._FakeSocket(response))
r.begin()
self.location = r.getheader('location')
self.usn = r.getheader('usn')
self.st = r.getheader('st')
self.cache = r.getheader('cache-control').split('=')[1]
def __init__(self, sock):
r = HTTPResponse(sock)
r.begin()
self.location = r.getheader("location")
self.usn = r.getheader("usn")
self.st = r.getheader("st")
self.cache = r.getheader("cache-control").split("=")[1]
def test_HTTPError():
fp = HTTPResponse(Sock(RESP), method='GET', url='http://foo.bar')
fp.begin()
e = HTTPError(
'http://foo.bar',
200,
'OK',
{'Foo': 'bar'},
fp,
)
assert isinstance(e.hdrs, Headers)
assert e.raw is fp
assert hasattr(e, 'encoding')
def _response_from_bytes(data: bytes) -> Urllib3HTTPResponse:
class BytesIOSocket:
def __init__(self, content):
self.handle = BytesIO(content)
def makefile(self, mode) -> BytesIO:
return self.handle
sock = BytesIOSocket(data)
response = HttpHTTPResponse(sock)
response.begin()
return Urllib3HTTPResponse.from_httplib(response)
def parse_response(data):
"""
Parses a saved HTTP response trace, e.g. "HTTP/1.1 200 OK\r\n\r\n{\"errors\":[]}"
"""
class BytesIOSocket:
def __init__(self, content):
self.handle = BytesIO(content)
def makefile(self, mode):
return self.handle
response = HTTPResponse(BytesIOSocket(data.encode("utf-8")))
response.begin()
return urllib3.HTTPResponse.from_httplib(response)
def begin(self):
return_value = HTTPResponse.begin(self)
if self.debuglevel == -1:
# Python 2
if hasattr(self.msg, 'headers'):
headers = [line.rstrip() for line in self.msg.headers]
# Python 3
else:
headers = []
for header in self.msg:
headers.append("%s: %s" % (header, self.msg[header]))
versions = {
9: u'HTTP/0.9',
10: u'HTTP/1.0',
11: u'HTTP/1.1'
}
status_line = u'%s %s %s' % (versions[self.version], str_cls(self.status), self.reason)
headers.insert(0, status_line)
indented_headers = u'\n '.join(headers)
console_write(
u'''
Urllib %s Debug Read
%s
''',
(self._debug_protocol, indented_headers)
)
return return_value
def __init__(self, sock):
r = HTTPResponse(sock)
r.begin()
self.location = r.getheader("location")
self.usn = r.getheader("usn")
self.st = r.getheader("st")
self.cache = r.getheader("cache-control").split("=")[1]
def __init__(self, response):
r = HTTPResponse(self._FakeSocket(response))
r.begin()
self.location = r.getheader('location')
self.usn = r.getheader('usn')
self.st = r.getheader('st')
self.cache = r.getheader('cache-control').split('=')[1]
def _extract_first_header_value(response: HTTPResponse, header_name: str) -> Optional[str]:
raw_header = response.getheader(header_name, None)
if not raw_header:
return None
# Handle headers defined multiple times by picking the first value
if ',' in raw_header:
raw_header = raw_header.split(',')[0]
return raw_header
def parse(sock):
# type: (socket) -> HTTPResponse
try:
# H4ck to standardize the API between sockets and SSLConnection objects
response = sock.read(4096)
except AttributeError:
response = sock.recv(4096)
while b'HTTP/' not in response or b'\r\n\r\n' not in response:
# Parse until the end of the headers
try:
response += sock.read(4096)
except AttributeError:
response += sock.recv(4096)
fake_sock = FakeSocket(response)
response = HTTPResponse(fake_sock)
response.begin()
return response
def handshake_with_server(reader, writer, parsed_url):
try:
rand = bytes(random.getrandbits(8) for _ in range(16))
key = base64.b64encode(rand).decode()
values = ''
if parsed_url.query:
values = '?' + parsed_url.query
handshake = _REQUEST % {'path': parsed_url.path + values, 'host_port': parsed_url.netloc, 'key': key}
writer.write(handshake.encode('utf-8'))
yield from writer.drain()
header_buffer = bytearray()
while True:
header = yield from reader.readline()
if len(header) == 0:
raise ProtocolError('no data from endpoint')
header_buffer.extend(header)
if len(header_buffer) > _WEBSOCKET_MAX_HEADER:
raise ProtocolError('header too large')
if header in b'\r\n':
break
response = HTTPResponse(FakeSocket(header_buffer))
response.begin()
accept_key = response.getheader('sec-websocket-accept')
if key is None:
raise ProtocolError('Sec-WebSocket-Accept does not exist')
digested_key = base64.b64encode(hashlib.sha1((key + _GUID_STRING).encode('utf-8')).digest())
if accept_key.encode() != digested_key:
raise ProtocolError('Sec-WebSocket-Accept key does not match')
return response
except asyncio.CancelledError:
writer.close()
def begin(self):
return_value = HTTPResponse.begin(self)
if self.debuglevel == -1:
console_write(u'Urllib %s Debug Read' % self._debug_protocol, True)
headers = self.msg.headers
versions = {
9: 'HTTP/0.9',
10: 'HTTP/1.0',
11: 'HTTP/1.1'
}
status_line = versions[self.version] + ' ' + str(self.status) + ' ' + self.reason
headers.insert(0, status_line)
for line in headers:
console_write(u" %s" % line.rstrip())
return return_value
def begin(self):
return_value = HTTPResponse.begin(self)
if self.debuglevel == -1:
console_write(u"Urllib %s Debug Read" % self._debug_protocol, True)
# Python 2
if hasattr(self.msg, "headers"):
headers = self.msg.headers
# Python 3
else:
headers = []
for header in self.msg:
headers.append("%s: %s" % (header, self.msg[header]))
versions = {9: "HTTP/0.9", 10: "HTTP/1.0", 11: "HTTP/1.1"}
status_line = versions[self.version] + " " + str(self.status) + " " + self.reason
headers.insert(0, status_line)
for line in headers:
console_write(u" %s" % line.rstrip())
return return_value
def __init__(self, request,proxy_socket):
HttpTransfer.__init__(self)
self.request = request
h = HTTPResponse(proxy_socket)
h.begin()
##HTTPResponse会将所有chunk拼接到一起，因此会直接得到所有内容，所以不能有Transfer-Encoding
del h.msg['Transfer-Encoding']
del h.msg['Content-Length']
self.response_version =self.version_dict[h.version]
self.status = h.status
self.reason = h.reason
self.set_headers(h.msg)
body_data = self._decode_content_body(h.read(),self.get_header('Content-Encoding'))
self.set_body_data(body_data)
self._text()#尝试将文本进行解码
h.close()
proxy_socket.close()
def read(self, *args):
try:
return HTTPResponse.read(self, *args)
except (IncompleteRead) as e:
return e.partial
def _read_status(self):
s = self.fp.read()
print('-' * 20, 'Response', '-' * 20)
print(s.split(b'\r\n\r\n')[0].decode('ascii'))
self.fp = io.BytesIO(s)
return HTTPResponse._read_status(self)
def __init__(self, sock, debuglevel=0, strict=0, method=None, buffering=False): HTTPResponse.__init__(self, sock, debuglevel, strict, method, buffering) self._refcount = 0