def from_raw(raw): # Raw is the output of curl, which is a full HTTP response. # But in the case of a redirect, it is multiple concatenated responses. # We want the final response, so we keep constructing new responses from this stream until we have reached the end while True: sock = FakeSocket(raw) response = HTTPResponse(sock) response.begin() response_len = sock.file.tell() + response.length raw_len = len(raw) if raw_len == response_len: break raw = raw[response_len:] raw_body = response.read() return Response( response.status, body=RawResponseBody(raw_body), seqno=int_or_none(response.getheader(CCF_TX_SEQNO_HEADER)), view=int_or_none(response.getheader(CCF_TX_VIEW_HEADER)), global_commit=int_or_none( response.getheader(CCF_GLOBAL_COMMIT_HEADER)), headers=response.headers, )
def from_raw(raw): sock = FakeSocket(raw) response = HTTPResponse(sock) response.begin() raw_body = response.read(raw) content_type = response.headers.get("content-type") if content_type == "application/json": parsed_body = json.loads(raw_body) elif content_type == "text/plain": parsed_body = raw_body.decode() elif content_type is None: parsed_body = None else: raise ValueError(f"Unhandled content type: {content_type}") return Response( response.status, body=parsed_body, seqno=int_or_none(response.getheader(CCF_TX_SEQNO_HEADER)), view=int_or_none(response.getheader(CCF_TX_VIEW_HEADER)), global_commit=int_or_none( response.getheader(CCF_GLOBAL_COMMIT_HEADER)), headers=response.headers, )
def from_raw(raw): sock = FakeSocket(raw) response = HTTPResponse(sock) response.begin() raw_body = response.read(raw) ok = response.status == 200 content_type = response.headers.get("content-type") if content_type == "application/json": parsed_body = json.loads(raw_body) elif content_type == "text/plain": parsed_body = raw_body.decode() elif content_type is None: parsed_body = None else: raise ValueError(f"Unhandled content type: {content_type}") return Response( status=response.status, result=parsed_body if ok else None, error=None if ok else parsed_body, commit=int_or_none(response.getheader(CCF_COMMIT_HEADER)), term=int_or_none(response.getheader(CCF_TERM_HEADER)), global_commit=int_or_none( response.getheader(CCF_GLOBAL_COMMIT_HEADER)), )
def __init__(self, sock): r = HTTPResponse(sock) r.begin() self.location = r.getheader("location") self.usn = r.getheader("usn") self.st = r.getheader("st") self.cache = r.getheader("cache-control").split("=")[1]
def __init__(self, sock): r = HTTPResponse(sock) r.begin() self.location = r.getheader("location") self.usn = r.getheader("usn") self.st = r.getheader("st") self.cache = r.getheader("cache-control").split("=")[1]
def __init__(self, response): r = HTTPResponse(self._FakeSocket(response)) r.begin() self.location = r.getheader('location') self.usn = r.getheader('usn') self.st = r.getheader('st') self.cache = r.getheader('cache-control').split('=')[1]
def __init__(self, response): r = HTTPResponse(self._FakeSocket(response)) r.begin() self.location = r.getheader('location') self.usn = r.getheader('usn') self.st = r.getheader('st') self.cache = r.getheader('cache-control').split('=')[1]
def from_raw(raw): sock = FakeSocket(raw) response = HTTPResponse(sock) response.begin() raw_body = response.read(raw) ok = response.status == 200 return Response( status=response.status, result=json.loads(raw_body) if ok else None, error=None if ok else raw_body.decode(), commit=int_or_none(response.getheader(CCF_COMMIT_HEADER)), term=int_or_none(response.getheader(CCF_TERM_HEADER)), global_commit=int_or_none(response.getheader(CCF_GLOBAL_COMMIT_HEADER)), )
async def start_connection(self, addr): sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) sock.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, 2) sock.bind((addr, 0)) # HTTP request message = "\r\n".join([ "M-SEARCH * HTTP/1.1", "HOST: 239.255.255.250:1900", 'MAN: "ssdp:discover"', "ST: urn:schemas-sony-com:service:ScalarWebAPI:1", "MX: 1", "", "", ]) print("Using interface %s" % addr) # Make sure the SSDP server is queried try: for _ in range(2): sock.sendto(message.encode(), ("239.255.255.250", 1900)) except: print("Failed to query SSDP server") try: data = sock.recv(1024) response = HTTPResponse(FakeSocket(data)) response.begin() st = response.getheader("st") if st != "urn:schemas-sony-com:service:ScalarWebAPI:1": return False self.location = response.getheader("location") print("Camera location header: %s" % self.location) await self.parse_ssdp_response() self.connected = True print("Connected to camera") return True except socket.timeout: print("Connection timed out") return False # Error while parsing http response except: traceback.print_exc() return False
def from_raw(raw): sock = FakeSocket(raw) response = HTTPResponse(sock) response.begin() raw_body = response.read(raw) return Response( response.status, body=RawResponseBody(raw_body), seqno=int_or_none(response.getheader(CCF_TX_SEQNO_HEADER)), view=int_or_none(response.getheader(CCF_TX_VIEW_HEADER)), global_commit=int_or_none(response.getheader(CCF_GLOBAL_COMMIT_HEADER)), headers=response.headers, )
def guess_image_extensinyon(respyonse: HTTPResponse) -> Optional[str]: '''Guess the extensinyon of the meownloaded image file. Tries Content-type header, then URL path (nyo~ support for 'Content-Disposition' cos Safebooru doesn’t support it anynway). ''' extensinyon: Optional[str] = None mime = respyonse.getheader('Content-type') if mime: logging.debug("Content-type is " + mime) guessed = mimetypes.guess_extension(mime) if guessed: extensinyon = guessed else: logging.info("Content-type is empty nya :/") if not extensinyon: path: str = urllib.parse.urlsplit(respyonse.url).path componyents: typing.List[str] = posixpath.basename(path).split('.') if len(componyents) < 2: logging.debug("Couldn't guess extension nyon :\ ") # TODO: could guess from data at this point return None else: extensinyon = '.' + componyents[-1] if extensinyon in ('.jpg', '.jpe'): extensinyon='.jpeg' logging.debug("Guessed extensinyon: " + extensinyon) return(extensinyon)
def get(addr, url, cert_checksum, user_agent=None, type=None): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.settimeout(1) wrapped_socket = ssl.create_default_context().wrap_socket( sock, server_hostname=addr) try: wrapped_socket.connect((addr, 443)) except: response = None else: der_cert_bin = wrapped_socket.getpeercert(True) if sha256(der_cert_bin).hexdigest() != cert_checksum: raise Exception("Incorrect certificate checksum") request_header = b"GET " + url + b" HTTP/1.0\r\nHost: " + addr if user_agent: request_header += b"\r\nUser-Agent: " + user_agent request_header += b"\r\n\r\n" wrapped_socket.send(request_header) response = HTTPResponse(wrapped_socket) response.begin() if type == "json": if response.getheader( "Content-Type") != "application/json; charset=utf-8": raise Exception( "Content-Type isn't application/json; charset=utf-8") body = response.read() wrapped_socket.close() return response, body
def _extract_variables(self, data: bytes) -> Mapping[str, bytes]: """ Subclasses must implement this method, from the response, it parses it and returns a dictionary with variables and their respective values. All variables set in self.required_vars must be set in this method, or parse() will raise a FuzzowskiRuntimeError Args: data: The response bytes Returns: A dictionary with all required variables (and optionally others) """ response_vars = {} try: source = FakeSocket(data) response = HTTPResponse(source) response.begin() content_length = int(response.getheader('Content-Length')) body = response.read(content_length) json_body = json.loads(body) for var in self.required_vars + self.optional_vars: if var in json_body: response_vars[var] = bytes(json_body[var], encoding='utf-8') except json.decoder.JSONDecodeError: pass except Exception: raise return response_vars
def _decode_json_response(r:HTTPResponse) -> Any: """ Decode JSON HTTP response """ charset = r.getheader('charset', 'utf-8') if r.headers.get_content_type() != 'application/json': return None return json.loads(r.read().decode(charset))
def get_sock_location(self, sock): message = self.ssdp_req try: sock.sendto(message.encode(), ("239.255.255.250", 1900)) data = sock.recv(1024) except: print("Failed to query SSDP server.") return None res = HTTPResponse(FakeSocket(data)) res.begin() st = res.getheader("st") if st != "urn:schemas-sony-com:service:ScalarWebAPI:1": return None else: location = res.getheader("location") return location
def _extract_first_header_value(response: HTTPResponse, header_name: str) -> Optional[str]: raw_header = response.getheader(header_name, None) if not raw_header: return None # Handle headers defined multiple times by picking the first value if ',' in raw_header: raw_header = raw_header.split(',')[0] return raw_header
def _extract_first_header_value(response: HTTPResponse, header_name: str) -> Optional[str]: raw_header = response.getheader(header_name, None) if not raw_header: return None # Handle headers defined multiple times by picking the first value if "," in raw_header: raw_header = raw_header.split(",")[0] return raw_header
def _http_response_asserts(response: HTTPResponse, fuzz_data_logger): if response.status >= 500: fuzz_data_logger.log_fail("Status code higher or equal than 500!") if response.getheader("Content-Type") == "application/json": try: json.loads(response.read()) except ValueError: fuzz_data_logger.log_fail( "application/json body is not valid JSON structure")
def should_drop_response(response): response = HTTPResponse(FakeSocket(response)) response.begin() content_type = response.getheader('Content-Type') for forbidden_type in TYPES_TO_FILTER: if forbidden_type in content_type: return True data = response.read() if is_source_code(data): return True return False
def test_ok_response_mime_type_is_there(fake_socket): """Test that request has a proper mime_type.""" from server import response_ok if sys.version_info.major == 3: from http.client import HTTPResponse else: from httplib import HTTPResponse response_str = response_ok(b'htmlhtml', 'text/plain') source = fake_socket(response_str) response = HTTPResponse(source) response.begin() assert response.getheader('Content-Type') == 'text/plain'
def __decode_data(resp: HTTPResponse): type = resp.getheader('Content-Encoding') if 'gzip' == type: from io import BytesIO import gzip buff = BytesIO(resp.read()) f = gzip.GzipFile(fileobj=buff) return f.read().decode('utf-8') elif 'br' == type: import brotli f = brotli.decompress(resp.read()) return f.decode('utf-8') else: return resp.read().decode('utf-8') # 返回的数据为byte类型,使用utf-8解码
def get_content(response: HTTPResponse) -> str: """Get content from HTTP response. Handles gzipped content. :param HTTPResponse response: HTTP response :returns: HTTP response content :rtype: str """ content = response.read() if response.getheader('Content-encoding') == 'gzip': content = gzip.decompress(content) return content.decode()
def get_filename(connection: HTTPResponse): content_disposition = connection.getheader('Content-Disposition') if content_disposition is not None: # !!!! I have yet to find a server that returns this for a file download. I have not been able to test this on # a proper response. I have made mock responses to test this. # Based off of https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Disposition dispositions = content_disposition.strip().split("; ") for disposition in dispositions: if "=" in disposition: key, value = disposition.split("=") if key == 'filename': return value # Will return '' if the path is '/path/to/something return Path(urlparse(connection.geturl()).path).name
class HTTPResponseWrapper(object): """ Class that adds some additional utilities to the build in HTTPResponse class. """ def __init__(self, oSock, sHttpVer): """ :param oSock: Communication socket. :param sHttpVer: Version of the http protocol we are using. """ self.oHTTPResponse = HTTPResponse(oSock) self.oHTTPResponse.begin() self.bBody = self.oHTTPResponse.read() self.bResponseLine = '{0} {1} {2}\r\n'.format( sHttpVer, self.oHTTPResponse.status, self.oHTTPResponse.reason).encode() self.bHeaders = self.oHTTPResponse.msg.as_bytes() self.sEncoding = self.oHTTPResponse.getheader('Content-Encoding') self.sContentType = self.oHTTPResponse.getheader('Content-Type') self.oHTTPResponse.close() def __bytes__(self): """ :return: Raw bytes of the http response. """ return self.bResponseLine + self.bHeaders + self.bBody def getDecodedBody(self): """ :return: Decoded content body of the http response. """ # Only gzip encoding and html content types are supported at the moment. if self.sEncoding == CONTENT_ENCODINGS.GZIP and CONTENT_TYPES.HTML in self.sContentType: try: # Zlib lets us decompress the gzipped contents as stored bytes. return zlib.decompress(self.bBody, 16 + zlib.MAX_WBITS) except Exception as e: print("Error decompressing gzip contents {0}".format(str(e)))
def _get_ip(res: HTTPResponse) -> Union[str, None]: loc = res.getheader("Location") if loc is not None: # it's a redirect, check to see if there's an IP in it parsed = urlparse(loc) domain = utils.get_domain(parsed.netloc) if utils.is_ip(domain): # it's an IP, now, is it private? if utils.is_private_ip(domain): return domain else: return None return None
def handshake_with_server(reader, writer, parsed_url, **kwds): max_header = kwds.get('max_header', 65536) try: rand = bytes(random.getrandbits(8) for _ in range(16)) key = base64.b64encode(rand).decode() values = '' if parsed_url.query: values = '?{0}'.format(parsed_url.query) handshake = _REQUEST % { 'path': parsed_url.path + values, 'host_port': parsed_url.netloc, 'key': key } writer.write(handshake.encode('utf-8')) yield from writer.drain() header_buffer = bytearray() while True: header = yield from reader.readline() if len(header) == 0: raise ProtocolError('no data from endpoint') header_buffer.extend(header) if len(header_buffer) > max_header: raise ProtocolError('header too large') if header in b'\r\n': break response = HTTPResponse(FakeSocket(header_buffer)) response.begin() accept_key = response.getheader('sec-websocket-accept') if key is None: raise ProtocolError('Sec-WebSocket-Accept does not exist') digested_key = base64.b64encode( hashlib.sha1((key + _GUID_STRING).encode('utf-8')).digest()) if accept_key.encode() != digested_key: raise ProtocolError('Sec-WebSocket-Accept key does not match') return response except asyncio.CancelledError: writer.close()
def test_error_response_405_well_formatted(fake_socket): """Test that error reponse of 405 HTTP response is correct.""" from server import response_error from datetime import datetime as time if sys.version_info.major == 3: from http.client import HTTPResponse else: from httplib import HTTPResponse source = fake_socket(response_error(405, 'Method Not Allowed')) response = HTTPResponse(source) response.begin() assert response.status == 405 assert time.strptime(response.getheader('Date'), '%a, %d %b %Y %H:%M:%S %Z')
def test_ok_response_well_formatted(fake_socket): """Test that formatting of 200 HTTP response is correct.""" from server import response_ok from datetime import datetime as time if sys.version_info.major == 3: from http.client import HTTPResponse else: from httplib import HTTPResponse source = fake_socket(response_ok(b'', 'text/plain')) response = HTTPResponse(source) response.begin() assert response.status == 200 assert time.strptime(response.getheader('Date'), '%a, %d %b %Y %H:%M:%S %Z')
def handshake_with_server(reader, writer, parsed_url): try: rand = bytes(random.getrandbits(8) for _ in range(16)) key = base64.b64encode(rand).decode() values = '' if parsed_url.query: values = '?' + parsed_url.query handshake = _REQUEST % {'path': parsed_url.path + values, 'host_port': parsed_url.netloc, 'key': key} writer.write(handshake.encode('utf-8')) yield from writer.drain() header_buffer = bytearray() while True: header = yield from reader.readline() if len(header) == 0: raise ProtocolError('no data from endpoint') header_buffer.extend(header) if len(header_buffer) > _WEBSOCKET_MAX_HEADER: raise ProtocolError('header too large') if header in b'\r\n': break response = HTTPResponse(FakeSocket(header_buffer)) response.begin() accept_key = response.getheader('sec-websocket-accept') if key is None: raise ProtocolError('Sec-WebSocket-Accept does not exist') digested_key = base64.b64encode(hashlib.sha1((key + _GUID_STRING).encode('utf-8')).digest()) if accept_key.encode() != digested_key: raise ProtocolError('Sec-WebSocket-Accept key does not match') return response except asyncio.CancelledError: writer.close()
def from_raw(raw): # Raw is the output of curl, which is a full HTTP response. # But in the case of a redirect, it is multiple concatenated responses. # We want the final response, so we keep constructing new responses from this stream until we have reached the end while True: sock = FakeSocket(raw) response = HTTPResponse(sock) response.begin() response_len = sock.file.tell() + response.length raw_len = len(raw) if raw_len == response_len: break raw = raw[response_len:] raw_body = response.read() tx_id = TxID.from_str(response.getheader(CCF_TX_ID_HEADER)) return Response( response.status, body=RawResponseBody(raw_body), seqno=tx_id.seqno, view=tx_id.view, headers=response.headers, )
def is_incomplete_query(response: HTTPResponse) -> bool: incomplete = response.getheader('X-SPARQL-MaxRows') != None if incomplete: raise SPARQLQueryTooLarge return incomplete
def is_partial_query(response: HTTPResponse) -> bool: partial = response.getheader('X-SQL-State') != None if partial: log.warn('Partial response!') return partial