def check_appid_exists(appid):
request_params, http_util, connection_cache_key = _get_request_params(
appid)
for _ in range(3):
err = None
response = None
try:
sock = http_util.create_ssl_connection(
(request_params.host, request_params.port),
request_params.hostname, connection_cache_key)
sock.sendall(b'HEAD / HTTP/1.1\r\n'
b'Host: %s\r\n'
b'Connection: Close\r\n\r\n' % host.encode())
response = HTTPResponse(sock, method='HEAD')
response.begin()
except:
err = True
finally:
if response:
response.close()
if err is None:
exists = response.status in (200, 503)
if exists:
http_util.ssl_connection_cache[
connection_cache_key].append((time(), sock))
return exists
def test_chunked_w_trailer() -> str:
client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
client.connect((config.SERVER_ADDR, config.SERVER_PORT))
request_header = "POST /post/print.cgi HTTP/1.1\r\nHost: localhost\r\nTransfer-encoding: chunked\r\n\r\n"
client.send(request_header.encode())
request_header = "5\r\ntest\n\r\n"
client.send(request_header.encode())
request_header = "0\r\naccept-language: fr\r\ntest-header: blabla\r\n\r\n"
client.send(request_header.encode())
# read and parse http response
http_response = HTTPResponse(client)
http_response.begin()
if http_response.status != 226:
return "Bad status code: {}, expected: {}".format(
str(http_response.status), "226")
if http_response.headers["Content-Type"] != "CGI/MINE":
return "Bad content-type: {}, expected: {}".format(
http_response.headers["Content-Type"], "CGI/MINE")
body = http_response.read().decode("UTF-8")
# print(body)
if (body.find("HTTP_TEST_HEADER=blabla") == -1
or body.find("HTTP_ACCEPT_LANGUAGE=fr") == -1):
return "Missing headers from request"
if body.find("test") == -1:
return "Missing content in request"
# print(body)
return ""
def __init__(self, request, proxy_socket):
HttpTransfer.__init__(self)
self.request = request
h = HTTPResponse(proxy_socket)
h.begin()
# HTTPResponse会将所有chunk拼接到一起，因此会直接得到所有内容，所以不能有Transfer-Encoding
del h.msg['Transfer-Encoding']
del h.msg['Content-Length']
self.response_version = self.version_dict[h.version]
self.status = h.status
self.reason = h.reason
self.set_headers(h.msg)
self.decoding = None
self.language = self.system = self.webserver = None
try:
data = h.read()
body_data = self._decode_content_body(
data, self.get_header('Content-Encoding'))
except http.client.IncompleteRead:
body_data = b''
except zlib.error:
body_data = b''
except _socket.timeout:
body_data = b''
self.set_body_data(body_data)
self._text() # 尝试将文本进行解码
h.close()
proxy_socket.close()
def from_string(data):
"""Parse http response to status code, headers and body
Based on: https://stackoverflow.com/questions/24728088/python-parse-http-response-string
"""
class FakeSocket:
"""Fake socket to simulate received http response content"""
def __init__(self, response_str):
self._file = BytesIO(response_str.encode('utf-8'))
def makefile(self, *args, **kwargs):
"""Fake file that provides string content"""
# pylint: disable=unused-argument
return self._file
source = FakeSocket(data)
response = HTTPResponse(source)
response.begin()
return ODataHttpResponse(
response.getheaders(),
response.status,
response.read(len(data)) # the len here will give a 'big enough' value to read the whole content
)
def _extract_variables(self, data: bytes) -> Mapping[str, bytes]:
"""
Subclasses must implement this method, from the response, it parses it and returns a dictionary with variables
and their respective values. All variables set in self.required_vars must be set in this method, or parse()
will raise a FuzzowskiRuntimeError
Args:
data: The response bytes
Returns: A dictionary with all required variables (and optionally others)
"""
response_vars = {}
try:
source = FakeSocket(data)
response = HTTPResponse(source)
response.begin()
content_length = int(response.getheader('Content-Length'))
body = response.read(content_length)
json_body = json.loads(body)
for var in self.required_vars + self.optional_vars:
if var in json_body:
response_vars[var] = bytes(json_body[var],
encoding='utf-8')
except json.decoder.JSONDecodeError:
pass
except Exception:
raise
return response_vars
def from_raw(raw):
sock = FakeSocket(raw)
response = HTTPResponse(sock)
response.begin()
raw_body = response.read(raw)
content_type = response.headers.get("content-type")
if content_type == "application/json":
parsed_body = json.loads(raw_body)
elif content_type == "text/plain":
parsed_body = raw_body.decode()
elif content_type is None:
parsed_body = None
else:
raise ValueError(f"Unhandled content type: {content_type}")
return Response(
response.status,
body=parsed_body,
seqno=int_or_none(response.getheader(CCF_TX_SEQNO_HEADER)),
view=int_or_none(response.getheader(CCF_TX_VIEW_HEADER)),
global_commit=int_or_none(
response.getheader(CCF_GLOBAL_COMMIT_HEADER)),
headers=response.headers,
)
def __init__(self, sock):
r = HTTPResponse(sock)
r.begin()
self.location = r.getheader("location")
self.usn = r.getheader("usn")
self.st = r.getheader("st")
self.cache = r.getheader("cache-control").split("=")[1]
def from_raw(raw):
sock = FakeSocket(raw)
response = HTTPResponse(sock)
response.begin()
raw_body = response.read(raw)
ok = response.status == 200
content_type = response.headers.get("content-type")
if content_type == "application/json":
parsed_body = json.loads(raw_body)
elif content_type == "text/plain":
parsed_body = raw_body.decode()
elif content_type is None:
parsed_body = None
else:
raise ValueError(f"Unhandled content type: {content_type}")
return Response(
status=response.status,
result=parsed_body if ok else None,
error=None if ok else parsed_body,
commit=int_or_none(response.getheader(CCF_COMMIT_HEADER)),
term=int_or_none(response.getheader(CCF_TERM_HEADER)),
global_commit=int_or_none(
response.getheader(CCF_GLOBAL_COMMIT_HEADER)),
)
def from_raw(raw):
# Raw is the output of curl, which is a full HTTP response.
# But in the case of a redirect, it is multiple concatenated responses.
# We want the final response, so we keep constructing new responses from this stream until we have reached the end
while True:
sock = FakeSocket(raw)
response = HTTPResponse(sock)
response.begin()
response_len = sock.file.tell() + response.length
raw_len = len(raw)
if raw_len == response_len:
break
raw = raw[response_len:]
raw_body = response.read()
return Response(
response.status,
body=RawResponseBody(raw_body),
seqno=int_or_none(response.getheader(CCF_TX_SEQNO_HEADER)),
view=int_or_none(response.getheader(CCF_TX_VIEW_HEADER)),
global_commit=int_or_none(
response.getheader(CCF_GLOBAL_COMMIT_HEADER)),
headers=response.headers,
)
def response_from_bytes(data):
sock = BytesIOSocket(data)
http_response = HTTPResponse(sock)
http_response.begin()
return urllib3.HTTPResponse.from_httplib(http_response)
def get(addr, url, cert_checksum, user_agent=None, type=None):
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.settimeout(1)
wrapped_socket = ssl.create_default_context().wrap_socket(
sock, server_hostname=addr)
try:
wrapped_socket.connect((addr, 443))
except:
response = None
else:
der_cert_bin = wrapped_socket.getpeercert(True)
if sha256(der_cert_bin).hexdigest() != cert_checksum:
raise Exception("Incorrect certificate checksum")
request_header = b"GET " + url + b" HTTP/1.0\r\nHost: " + addr
if user_agent:
request_header += b"\r\nUser-Agent: " + user_agent
request_header += b"\r\n\r\n"
wrapped_socket.send(request_header)
response = HTTPResponse(wrapped_socket)
response.begin()
if type == "json":
if response.getheader(
"Content-Type") != "application/json; charset=utf-8":
raise Exception(
"Content-Type isn't application/json; charset=utf-8")
body = response.read()
wrapped_socket.close()
return response, body
def discover(timeout=2, retries=1, st=ST_ECP):
group = ('239.255.255.250', 1900)
message = '\r\n'.join([
'M-SEARCH * HTTP/1.1',
'HOST: {0}:{1}'.format(*group),
'MAN: "ssdp:discover"',
'ST: {st}', 'MX: 3', '', ''])
socket.setdefaulttimeout(timeout)
responses = {}
for _ in range(retries):
sock = socket.socket(
socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, 2)
m = message.format(st=st)
sock.sendto(m.encode(), group)
while 1:
try:
rhttp = HTTPResponse(_FakeSocket(sock.recv(1024)))
rhttp.begin()
if rhttp.status == 200:
rssdp = SSDPResponse(rhttp)
responses[rssdp.location] = rssdp
except socket.timeout:
break
return responses.values()
def inner(code, location):
r = HTTPResponse(
Sock(RESP % (code, codes[code].encode(), location.encode())),
method='GET',
url='https://foo.bar/',
)
r.begin()
return r
def do_COMMAND(self):
log.info(f"{self.command} {self.path}")
try:
# Connect to destination
self._connect_to_host()
except Exception as e:
self.send_error(500, str(e))
return
# The request that got sent to the website: self
# Browser <--> [Proxy <--> Website]
content_length = int(self.headers.get("Content-Length", 0))
request_body = self.rfile.read(content_length)
# Build request which will be sent to the client
# [Browser <--> Proxy] <--> Website
client_request = (
# Add "GET / HTTP/1.1..." to the request"
b" ".join([
to_bytes(self.command),
to_bytes(self.path),
to_bytes(self.request_version),
]) + CRLF
# Add Headers to the request (Host:..., User-Agent:...)
+ self.headers.as_bytes() + CRLF + request_body)
# Send it down the pipe!
self.server.send(to_bytes(client_request))
# Parse response
h = HTTPResponse(self.server.conn)
h.begin()
# Get rid of hop-by-hop headers
orig_response = h
self.filter_headers(h.msg)
# Time to relay the message across
# read response body
response_body = h.read()
res = (
# HTTP/1.1 OK
b" ".join([
to_bytes(self.request_version),
to_bytes(h.status),
to_bytes(h.reason),
])
# Content-Type, Content-Length, Server...
+ CRLF + h.msg.as_bytes() + CRLF + response_body)
# Let's close off the remote end
h.close()
self.server.close()
# Relay the message
self.client.send(res)
self.print_info(self, request_body, orig_response, response_body)
def parse_response(self, response_text):
"""
Given a raw HTTP response, create a httplib.HTTResponse out of it.
"""
source = FakeSocket(response_text)
response = HTTPResponse(source)
response.begin()
res = response.read()
return (res, response.status)
def read_response(self, sock):
response = HTTPResponse(sock)
response.begin()
content = response.read()
status_code = response.status
reason = response.reason
headers = dict(response.getheaders())
response.close()
return FakeResponse(status_code, reason, headers, content)
def _request(self, sock, method, path, protocol_version, headers, payload, bufsize=8192):
request_data = []
request_data.append('%s %s %s' % (method, path, protocol_version))
for k, v in headers.items():
request_data.append('%s: %s' % (k.title(), v))
#if self.proxy:
# _, username, password, _ = parse_proxy(self.proxy)
# if username and password:
# request_data += 'Proxy-Authorization: Basic %s\r\n' % base64.b64encode(('%s:%s' % (username, password)).encode()).decode().strip()
request_data.append('\r\n')
if hasattr(payload, 'read'):
#避免发送多个小数据包
sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, False)
request_data = '\r\n'.join(request_data).encode()
sock.sendall(request_data)
try:
#以下按原样转发
readed = 0
left_size = int(headers.get('Content-Length', 0))
while True:
if left_size < 1:
break
data = payload.read(min(bufsize, left_size))
sock.sendall(data)
left_size -= len(data)
readed += len(data)
finally:
if payload.__class__.__name__ == '_PaddedFile':
payload.file.readed = readed
else:
payload.readed = readed
#为下个请求恢复无延迟发送
sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
else:
request_data = '\r\n'.join(request_data).encode() + payload
sock.sendall(request_data)
# try:
# response = HTTPResponse(sock, method=method)
# response.begin()
# except Exception as e:
#这里有时会捕捉到奇怪的异常，找不到来源路径
# py2 的 raise 不带参数会导致捕捉到错误的异常，但使用 exc_clear 或换用 py3 还是会出现
# if hasattr(e, 'xip'):
#logging.warning('4444 %r | %r | %r', sock.getpeername(), sock.xip, e.xip)
# del e.xip
# raise e
response = HTTPResponse(sock, method=method)
response.begin()
response.xip = sock.xip
response.sock = sock
return response
def _parse(read_method: Callable) -> HTTPResponse:
"""Trick to standardize the API between sockets and SSLConnection objects.
"""
response = read_method(4096)
while b'HTTP/' not in response or b'\r\n\r\n' not in response:
# Parse until the end of the headers
response += read_method(4096)
fake_sock = _FakeSocket(response)
response = HTTPResponse(fake_sock) # type: ignore
response.begin()
return response
