def from_raw(raw):
# Raw is the output of curl, which is a full HTTP response.
# But in the case of a redirect, it is multiple concatenated responses.
# We want the final response, so we keep constructing new responses from this stream until we have reached the end
while True:
sock = FakeSocket(raw)
response = HTTPResponse(sock)
response.begin()
response_len = sock.file.tell() + response.length
raw_len = len(raw)
if raw_len == response_len:
break
raw = raw[response_len:]
raw_body = response.read()
return Response(
response.status,
body=RawResponseBody(raw_body),
seqno=int_or_none(response.getheader(CCF_TX_SEQNO_HEADER)),
view=int_or_none(response.getheader(CCF_TX_VIEW_HEADER)),
global_commit=int_or_none(
response.getheader(CCF_GLOBAL_COMMIT_HEADER)),
headers=response.headers,
)
def from_raw(raw):
sock = FakeSocket(raw)
response = HTTPResponse(sock)
response.begin()
raw_body = response.read(raw)
content_type = response.headers.get("content-type")
if content_type == "application/json":
parsed_body = json.loads(raw_body)
elif content_type == "text/plain":
parsed_body = raw_body.decode()
elif content_type is None:
parsed_body = None
else:
raise ValueError(f"Unhandled content type: {content_type}")
return Response(
response.status,
body=parsed_body,
seqno=int_or_none(response.getheader(CCF_TX_SEQNO_HEADER)),
view=int_or_none(response.getheader(CCF_TX_VIEW_HEADER)),
global_commit=int_or_none(
response.getheader(CCF_GLOBAL_COMMIT_HEADER)),
headers=response.headers,
)
def from_raw(raw):
sock = FakeSocket(raw)
response = HTTPResponse(sock)
response.begin()
raw_body = response.read(raw)
ok = response.status == 200
content_type = response.headers.get("content-type")
if content_type == "application/json":
parsed_body = json.loads(raw_body)
elif content_type == "text/plain":
parsed_body = raw_body.decode()
elif content_type is None:
parsed_body = None
else:
raise ValueError(f"Unhandled content type: {content_type}")
return Response(
status=response.status,
result=parsed_body if ok else None,
error=None if ok else parsed_body,
commit=int_or_none(response.getheader(CCF_COMMIT_HEADER)),
term=int_or_none(response.getheader(CCF_TERM_HEADER)),
global_commit=int_or_none(
response.getheader(CCF_GLOBAL_COMMIT_HEADER)),
)
def __init__(self, sock):
r = HTTPResponse(sock)
r.begin()
self.location = r.getheader("location")
self.usn = r.getheader("usn")
self.st = r.getheader("st")
self.cache = r.getheader("cache-control").split("=")[1]
def __init__(self, sock):
r = HTTPResponse(sock)
r.begin()
self.location = r.getheader("location")
self.usn = r.getheader("usn")
self.st = r.getheader("st")
self.cache = r.getheader("cache-control").split("=")[1]
def __init__(self, response):
r = HTTPResponse(self._FakeSocket(response))
r.begin()
self.location = r.getheader('location')
self.usn = r.getheader('usn')
self.st = r.getheader('st')
self.cache = r.getheader('cache-control').split('=')[1]
def __init__(self, response):
r = HTTPResponse(self._FakeSocket(response))
r.begin()
self.location = r.getheader('location')
self.usn = r.getheader('usn')
self.st = r.getheader('st')
self.cache = r.getheader('cache-control').split('=')[1]
def from_raw(raw):
sock = FakeSocket(raw)
response = HTTPResponse(sock)
response.begin()
raw_body = response.read(raw)
ok = response.status == 200
return Response(
status=response.status,
result=json.loads(raw_body) if ok else None,
error=None if ok else raw_body.decode(),
commit=int_or_none(response.getheader(CCF_COMMIT_HEADER)),
term=int_or_none(response.getheader(CCF_TERM_HEADER)),
global_commit=int_or_none(response.getheader(CCF_GLOBAL_COMMIT_HEADER)),
)
async def start_connection(self, addr):
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM,
socket.IPPROTO_UDP)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, 2)
sock.bind((addr, 0))
# HTTP request
message = "\r\n".join([
"M-SEARCH * HTTP/1.1",
"HOST: 239.255.255.250:1900",
'MAN: "ssdp:discover"',
"ST: urn:schemas-sony-com:service:ScalarWebAPI:1",
"MX: 1",
"",
"",
])
print("Using interface %s" % addr)
# Make sure the SSDP server is queried
try:
for _ in range(2):
sock.sendto(message.encode(), ("239.255.255.250", 1900))
except:
print("Failed to query SSDP server")
try:
data = sock.recv(1024)
response = HTTPResponse(FakeSocket(data))
response.begin()
st = response.getheader("st")
if st != "urn:schemas-sony-com:service:ScalarWebAPI:1":
return False
self.location = response.getheader("location")
print("Camera location header: %s" % self.location)
await self.parse_ssdp_response()
self.connected = True
print("Connected to camera")
return True
except socket.timeout:
print("Connection timed out")
return False
# Error while parsing http response
except:
traceback.print_exc()
return False
def from_raw(raw):
sock = FakeSocket(raw)
response = HTTPResponse(sock)
response.begin()
raw_body = response.read(raw)
return Response(
response.status,
body=RawResponseBody(raw_body),
seqno=int_or_none(response.getheader(CCF_TX_SEQNO_HEADER)),
view=int_or_none(response.getheader(CCF_TX_VIEW_HEADER)),
global_commit=int_or_none(response.getheader(CCF_GLOBAL_COMMIT_HEADER)),
headers=response.headers,
)
def guess_image_extensinyon(respyonse: HTTPResponse) -> Optional[str]:
'''Guess the extensinyon of the meownloaded image file.
Tries Content-type header, then URL path (nyo~ support for
'Content-Disposition' cos Safebooru doesn’t support it anynway).
'''
extensinyon: Optional[str] = None
mime = respyonse.getheader('Content-type')
if mime:
logging.debug("Content-type is " + mime)
guessed = mimetypes.guess_extension(mime)
if guessed:
extensinyon = guessed
else:
logging.info("Content-type is empty nya :/")
if not extensinyon:
path: str = urllib.parse.urlsplit(respyonse.url).path
componyents: typing.List[str] = posixpath.basename(path).split('.')
if len(componyents) < 2:
logging.debug("Couldn't guess extension nyon :\ ")
# TODO: could guess from data at this point
return None
else:
extensinyon = '.' + componyents[-1]
if extensinyon in ('.jpg', '.jpe'):
extensinyon='.jpeg'
logging.debug("Guessed extensinyon: " + extensinyon)
return(extensinyon)
def get(addr, url, cert_checksum, user_agent=None, type=None):
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.settimeout(1)
wrapped_socket = ssl.create_default_context().wrap_socket(
sock, server_hostname=addr)
try:
wrapped_socket.connect((addr, 443))
except:
response = None
else:
der_cert_bin = wrapped_socket.getpeercert(True)
if sha256(der_cert_bin).hexdigest() != cert_checksum:
raise Exception("Incorrect certificate checksum")
request_header = b"GET " + url + b" HTTP/1.0\r\nHost: " + addr
if user_agent:
request_header += b"\r\nUser-Agent: " + user_agent
request_header += b"\r\n\r\n"
wrapped_socket.send(request_header)
response = HTTPResponse(wrapped_socket)
response.begin()
if type == "json":
if response.getheader(
"Content-Type") != "application/json; charset=utf-8":
raise Exception(
"Content-Type isn't application/json; charset=utf-8")
body = response.read()
wrapped_socket.close()
return response, body
def _extract_variables(self, data: bytes) -> Mapping[str, bytes]:
"""
Subclasses must implement this method, from the response, it parses it and returns a dictionary with variables
and their respective values. All variables set in self.required_vars must be set in this method, or parse()
will raise a FuzzowskiRuntimeError
Args:
data: The response bytes
Returns: A dictionary with all required variables (and optionally others)
"""
response_vars = {}
try:
source = FakeSocket(data)
response = HTTPResponse(source)
response.begin()
content_length = int(response.getheader('Content-Length'))
body = response.read(content_length)
json_body = json.loads(body)
for var in self.required_vars + self.optional_vars:
if var in json_body:
response_vars[var] = bytes(json_body[var],
encoding='utf-8')
except json.decoder.JSONDecodeError:
pass
except Exception:
raise
return response_vars
def _decode_json_response(r:HTTPResponse) -> Any:
""" Decode JSON HTTP response """
charset = r.getheader('charset', 'utf-8')
if r.headers.get_content_type() != 'application/json':
return None
return json.loads(r.read().decode(charset))
def get_sock_location(self, sock):
message = self.ssdp_req
try:
sock.sendto(message.encode(), ("239.255.255.250", 1900))
data = sock.recv(1024)
except:
print("Failed to query SSDP server.")
return None
res = HTTPResponse(FakeSocket(data))
res.begin()
st = res.getheader("st")
if st != "urn:schemas-sony-com:service:ScalarWebAPI:1":
return None
else:
location = res.getheader("location")
return location
def _extract_first_header_value(response: HTTPResponse, header_name: str) -> Optional[str]:
raw_header = response.getheader(header_name, None)
if not raw_header:
return None
# Handle headers defined multiple times by picking the first value
if ',' in raw_header:
raw_header = raw_header.split(',')[0]
return raw_header
def _extract_first_header_value(response: HTTPResponse, header_name: str) -> Optional[str]:
raw_header = response.getheader(header_name, None)
if not raw_header:
return None
# Handle headers defined multiple times by picking the first value
if "," in raw_header:
raw_header = raw_header.split(",")[0]
return raw_header
def _http_response_asserts(response: HTTPResponse, fuzz_data_logger):
if response.status >= 500:
fuzz_data_logger.log_fail("Status code higher or equal than 500!")
if response.getheader("Content-Type") == "application/json":
try:
json.loads(response.read())
except ValueError:
fuzz_data_logger.log_fail(
"application/json body is not valid JSON structure")
def should_drop_response(response):
response = HTTPResponse(FakeSocket(response))
response.begin()
content_type = response.getheader('Content-Type')
for forbidden_type in TYPES_TO_FILTER:
if forbidden_type in content_type:
return True
data = response.read()
if is_source_code(data):
return True
return False
def test_ok_response_mime_type_is_there(fake_socket):
"""Test that request has a proper mime_type."""
from server import response_ok
if sys.version_info.major == 3:
from http.client import HTTPResponse
else:
from httplib import HTTPResponse
response_str = response_ok(b'htmlhtml', 'text/plain')
source = fake_socket(response_str)
response = HTTPResponse(source)
response.begin()
assert response.getheader('Content-Type') == 'text/plain'
def __decode_data(resp: HTTPResponse):
type = resp.getheader('Content-Encoding')
if 'gzip' == type:
from io import BytesIO
import gzip
buff = BytesIO(resp.read())
f = gzip.GzipFile(fileobj=buff)
return f.read().decode('utf-8')
elif 'br' == type:
import brotli
f = brotli.decompress(resp.read())
return f.decode('utf-8')
else:
return resp.read().decode('utf-8') # 返回的数据为byte类型，使用utf-8解码
def get_content(response: HTTPResponse) -> str:
"""Get content from HTTP response.
Handles gzipped content.
:param HTTPResponse response: HTTP response
:returns: HTTP response content
:rtype: str
"""
content = response.read()
if response.getheader('Content-encoding') == 'gzip':
content = gzip.decompress(content)
return content.decode()
def get_filename(connection: HTTPResponse):
content_disposition = connection.getheader('Content-Disposition')
if content_disposition is not None:
# !!!! I have yet to find a server that returns this for a file download. I have not been able to test this on
# a proper response. I have made mock responses to test this.
# Based off of https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Disposition
dispositions = content_disposition.strip().split("; ")
for disposition in dispositions:
if "=" in disposition:
key, value = disposition.split("=")
if key == 'filename':
return value
# Will return '' if the path is '/path/to/something
return Path(urlparse(connection.geturl()).path).name
class HTTPResponseWrapper(object):
"""
Class that adds some additional utilities to the build in HTTPResponse class.
"""
def __init__(self, oSock, sHttpVer):
"""
:param oSock: Communication socket.
:param sHttpVer: Version of the http protocol we are using.
"""
self.oHTTPResponse = HTTPResponse(oSock)
self.oHTTPResponse.begin()
self.bBody = self.oHTTPResponse.read()
self.bResponseLine = '{0} {1} {2}\r\n'.format(
sHttpVer, self.oHTTPResponse.status,
self.oHTTPResponse.reason).encode()
self.bHeaders = self.oHTTPResponse.msg.as_bytes()
self.sEncoding = self.oHTTPResponse.getheader('Content-Encoding')
self.sContentType = self.oHTTPResponse.getheader('Content-Type')
self.oHTTPResponse.close()
def __bytes__(self):
"""
:return: Raw bytes of the http response.
"""
return self.bResponseLine + self.bHeaders + self.bBody
def getDecodedBody(self):
"""
:return: Decoded content body of the http response.
"""
# Only gzip encoding and html content types are supported at the moment.
if self.sEncoding == CONTENT_ENCODINGS.GZIP and CONTENT_TYPES.HTML in self.sContentType:
try:
# Zlib lets us decompress the gzipped contents as stored bytes.
return zlib.decompress(self.bBody, 16 + zlib.MAX_WBITS)
except Exception as e:
print("Error decompressing gzip contents {0}".format(str(e)))
def _get_ip(res: HTTPResponse) -> Union[str, None]:
loc = res.getheader("Location")
if loc is not None:
# it's a redirect, check to see if there's an IP in it
parsed = urlparse(loc)
domain = utils.get_domain(parsed.netloc)
if utils.is_ip(domain):
# it's an IP, now, is it private?
if utils.is_private_ip(domain):
return domain
else:
return None
return None
def handshake_with_server(reader, writer, parsed_url, **kwds):
max_header = kwds.get('max_header', 65536)
try:
rand = bytes(random.getrandbits(8) for _ in range(16))
key = base64.b64encode(rand).decode()
values = ''
if parsed_url.query:
values = '?{0}'.format(parsed_url.query)
handshake = _REQUEST % {
'path': parsed_url.path + values,
'host_port': parsed_url.netloc,
'key': key
}
writer.write(handshake.encode('utf-8'))
yield from writer.drain()
header_buffer = bytearray()
while True:
header = yield from reader.readline()
if len(header) == 0:
raise ProtocolError('no data from endpoint')
header_buffer.extend(header)
if len(header_buffer) > max_header:
raise ProtocolError('header too large')
if header in b'\r\n':
break
response = HTTPResponse(FakeSocket(header_buffer))
response.begin()
accept_key = response.getheader('sec-websocket-accept')
if key is None:
raise ProtocolError('Sec-WebSocket-Accept does not exist')
digested_key = base64.b64encode(
hashlib.sha1((key + _GUID_STRING).encode('utf-8')).digest())
if accept_key.encode() != digested_key:
raise ProtocolError('Sec-WebSocket-Accept key does not match')
return response
except asyncio.CancelledError:
writer.close()
def test_error_response_405_well_formatted(fake_socket):
"""Test that error reponse of 405 HTTP response is correct."""
from server import response_error
from datetime import datetime as time
if sys.version_info.major == 3:
from http.client import HTTPResponse
else:
from httplib import HTTPResponse
source = fake_socket(response_error(405, 'Method Not Allowed'))
response = HTTPResponse(source)
response.begin()
assert response.status == 405
assert time.strptime(response.getheader('Date'),
'%a, %d %b %Y %H:%M:%S %Z')
def test_ok_response_well_formatted(fake_socket):
"""Test that formatting of 200 HTTP response is correct."""
from server import response_ok
from datetime import datetime as time
if sys.version_info.major == 3:
from http.client import HTTPResponse
else:
from httplib import HTTPResponse
source = fake_socket(response_ok(b'', 'text/plain'))
response = HTTPResponse(source)
response.begin()
assert response.status == 200
assert time.strptime(response.getheader('Date'),
'%a, %d %b %Y %H:%M:%S %Z')
def handshake_with_server(reader, writer, parsed_url):
try:
rand = bytes(random.getrandbits(8) for _ in range(16))
key = base64.b64encode(rand).decode()
values = ''
if parsed_url.query:
values = '?' + parsed_url.query
handshake = _REQUEST % {'path': parsed_url.path + values, 'host_port': parsed_url.netloc, 'key': key}
writer.write(handshake.encode('utf-8'))
yield from writer.drain()
header_buffer = bytearray()
while True:
header = yield from reader.readline()
if len(header) == 0:
raise ProtocolError('no data from endpoint')
header_buffer.extend(header)
if len(header_buffer) > _WEBSOCKET_MAX_HEADER:
raise ProtocolError('header too large')
if header in b'\r\n':
break
response = HTTPResponse(FakeSocket(header_buffer))
response.begin()
accept_key = response.getheader('sec-websocket-accept')
if key is None:
raise ProtocolError('Sec-WebSocket-Accept does not exist')
digested_key = base64.b64encode(hashlib.sha1((key + _GUID_STRING).encode('utf-8')).digest())
if accept_key.encode() != digested_key:
raise ProtocolError('Sec-WebSocket-Accept key does not match')
return response
except asyncio.CancelledError:
writer.close()
def from_raw(raw):
# Raw is the output of curl, which is a full HTTP response.
# But in the case of a redirect, it is multiple concatenated responses.
# We want the final response, so we keep constructing new responses from this stream until we have reached the end
while True:
sock = FakeSocket(raw)
response = HTTPResponse(sock)
response.begin()
response_len = sock.file.tell() + response.length
raw_len = len(raw)
if raw_len == response_len:
break
raw = raw[response_len:]
raw_body = response.read()
tx_id = TxID.from_str(response.getheader(CCF_TX_ID_HEADER))
return Response(
response.status,
body=RawResponseBody(raw_body),
seqno=tx_id.seqno,
view=tx_id.view,
headers=response.headers,
)
def is_incomplete_query(response: HTTPResponse) -> bool:
incomplete = response.getheader('X-SPARQL-MaxRows') != None
if incomplete:
raise SPARQLQueryTooLarge
return incomplete
def is_partial_query(response: HTTPResponse) -> bool:
partial = response.getheader('X-SQL-State') != None
if partial:
log.warn('Partial response!')
return partial
