def get_response(self, request, connection): """ return final respons, it is only accessible via peform method """ if log.isEnabledFor(logging.DEBUG): log.debug("Start to parse response") p = HttpStream(SocketReader(connection.socket()), kind=1, decompress=self.decompress) if log.isEnabledFor(logging.DEBUG): log.debug("Got response: %s %s" % (p.version(), p.status())) log.debug("headers: [%s]" % p.headers()) location = p.headers().get('location') if self.follow_redirect: should_close = not p.should_keep_alive() if p.status_code() in (301, 302, 307,): # read full body and release the connection p.body_file().read() connection.release(should_close) if request.method in ('GET', 'HEAD',) or \ self.force_follow_redirect: if hasattr(self.body, 'read'): try: self.body.seek(0) except AttributeError: raise RequestError("Can't redirect %s to %s " "because body has already been read" % (self.url, location)) return self.redirect(location, request) elif p.status_code() == 303 and self.method == "POST": # read full body and release the connection p.body_file().read() connection.release(should_close) request.method = "GET" request.body = None return self.redirect(location, request) # create response object resp = self.response_class(connection, request, p) # apply response filters for f in self.response_filters: f.on_response(resp, request) if log.isEnabledFor(logging.DEBUG): log.debug("return response class") # return final response return resp
def get_response(self, request, connection): """ return final respons, it is only accessible via peform method """ if log.isEnabledFor(logging.DEBUG): log.debug("Start to parse response") p = HttpStream(SocketReader(connection.socket()), kind=1, decompress=self.decompress) if log.isEnabledFor(logging.DEBUG): log.debug("Got response: %s %s" % (p.version(), p.status())) log.debug("headers: [%s]" % p.headers()) location = p.headers().get('location') if self.follow_redirect: should_close = not p.should_keep_alive() if p.status_code() in (301, 302, 307,): # read full body and release the connection p.body_file().read() connection.release(should_close) if request.method in ('GET', 'HEAD',) or \ self.force_follow_redirect: if hasattr(self.body, 'read'): try: self.body.seek(0) except AttributeError: raise RequestError("Can't redirect %s to %s " "because body has already been read" % (self.url, location)) return self.redirect(location, request) elif p.status_code() == 303 and self.method == "POST": # read full body and release the connection p.body_file().read() connection.release(should_close) request.method = "GET" request.body = None return self.redirect(location, request) # create response object resp = self.response_class(connection, request, p) # apply response filters for f in self.response_filters: f.on_response(resp, request) if log.isEnabledFor(logging.DEBUG): log.debug("return response class") # return final response return resp
def read_get(self): reader = SocketReader(self.sock) resp = HttpStream(reader) status_code = resp.status_code() headers = dict(resp.headers()) body = resp.body_string() return {"status_code": status_code, "headers": headers, "body": body}
def get_response(self, request, connection): """ return final respons, it is only accessible via peform method """ if log.isEnabledFor(logging.DEBUG): log.debug("Start to parse response") p = HttpStream(SocketReader(connection.socket()), kind=1, decompress=self.decompress) if log.isEnabledFor(logging.DEBUG): log.debug("Got response: %s" % p.status()) log.debug("headers: [%s]" % p.headers()) location = p.headers().get("location") if self.follow_redirect: if p.status_code() in (301, 302, 307): if request.method in ("GET", "HEAD") or self.force_follow_redirect: if hasattr(self.body, "read"): try: self.body.seek(0) except AttributeError: connection.release() raise RequestError( "Can't redirect %s to %s " "because body has already been read" % (self.url, location) ) connection.release() return self.redirect(p, location, request) elif p.status_code() == 303 and self.method == "POST": connection.release() request.method = "GET" request.body = None return self.redirect(p, location, request) # create response object resp = self.response_class(connection, request, p) # apply response filters for f in self.response_filters: f.on_response(resp, request) if log.isEnabledFor(logging.DEBUG): log.debug("return response class") # return final response return resp
def recv(self): try: stream = HttpStream(self.reader, kind=HTTP_RESPONSE, parser_class=HttpParser, decompress=True) status = stream.status_code() version = stream.version() fields = stream.headers() content = stream.body_file() self.version = 'HTTP/%s.%s' % version return status, fields, content except NoMoreData: pass
def proxy_connection(self, request, req_addr, is_ssl): """ do the proxy connection """ proxy_settings = os.environ.get('%s_proxy' % request.parsed_url.scheme) if proxy_settings and proxy_settings is not None: request.is_proxied = True proxy_settings, proxy_auth = _get_proxy_auth(proxy_settings) addr = parse_netloc(urlparse.urlparse(proxy_settings)) if is_ssl: if proxy_auth: proxy_auth = 'Proxy-authorization: %s' % proxy_auth proxy_connect = 'CONNECT %s:%s HTTP/1.0\r\n' % req_addr user_agent = request.headers.iget('user_agent') if not user_agent: user_agent = "User-Agent: restkit/%s\r\n" % __version__ proxy_pieces = '%s%s%s\r\n' % (proxy_connect, proxy_auth, user_agent) conn = self._pool.get(host=addr[0], port=addr[1], pool=self._pool, is_ssl=is_ssl, extra_headers=[], **self.ssl_args) conn.send(proxy_pieces) p = HttpStream(SocketReader(conn.socket()), kind=1, decompress=True) if p.status_code != 200: raise ProxyError("Tunnel connection failed: %d %s" % (p.status_code(), p.body_string())) # read body p.body_string() else: headers = [] if proxy_auth: headers = [('Proxy-authorization', proxy_auth)] conn = self._pool.get(host=addr[0], port=addr[1], pool=self._pool, is_ssl=False, extra_headers=[], **self.ssl_args) return conn return
def proxy_connection(self, request, req_addr, is_ssl): """ do the proxy connection """ proxy_settings = os.environ.get('%s_proxy' % request.parsed_url.scheme) if proxy_settings and proxy_settings is not None: request.is_proxied = True proxy_settings, proxy_auth = _get_proxy_auth(proxy_settings) addr = parse_netloc(urlparse.urlparse(proxy_settings)) if is_ssl: if proxy_auth: proxy_auth = 'Proxy-authorization: %s' % proxy_auth proxy_connect = 'CONNECT %s:%s HTTP/1.0\r\n' % req_addr user_agent = request.headers.iget('user_agent') if not user_agent: user_agent = "User-Agent: restkit/%s\r\n" % __version__ proxy_pieces = '%s%s%s\r\n' % (proxy_connect, proxy_auth, user_agent) conn = self._pool.get(host=addr[0], port=addr[1], pool=self._pool, is_ssl=is_ssl, extra_headers=[], **self.ssl_args) conn.send(proxy_pieces) p = HttpStream(SocketReader(conn.socket()), kind=1, decompress=True) if p.status_code != 200: raise ProxyError("Tunnel connection failed: %d %s" % (p.status_code(), p.body_string())) # read body p.body_string() else: headers = [] if proxy_auth: headers = [('Proxy-authorization', proxy_auth)] conn = self._pool.get(host=addr[0], port=addr[1], pool=self._pool, is_ssl=False, extra_headers=[], **self.ssl_args) return conn return
def https_proxy(self, connectreq: HttpStream): self.request.sendall(CON_ESTABLISH) host, port = connectreq.path().split(':') cert_path = self.generate_cert(host) # request to localhost as server try: sock_client = ssl.wrap_socket(self.request, keyfile="cert.key", certfile=cert_path, ssl_version=ssl.PROTOCOL_TLS, server_side=True) sock_server = socket.create_connection((host, port)) sock_server = ssl.create_default_context().wrap_socket(sock_server, server_hostname=host) except OSError: return client_reader = SocketReader(sock_client) server_reader = SocketReader(sock_server) while True: # request to mail.ru as client clientreq = HttpStream(client_reader, decompress=True) try: clientreq_tosend = join_data(clientreq, 'Request') except (http_parser.http.ParserError, http_parser.http.BadStatusLine, ConnectionResetError): return sock_server.sendall(clientreq_tosend) # request from mail.ru to localhost as client server_resp = HttpStream(server_reader, decompress=True) try: server_resp_tosend = join_data(server_resp, 'Response') except (http_parser.http.ParserError, http_parser.http.BadStatusLine, ConnectionResetError): return try: sock_client.sendall(server_resp_tosend) except BrokenPipeError: return req = Request(clientreq.method(), server_resp.status_code(), clientreq_tosend, host) self.add_date_base(clientreq, req)
def http_proxy(self, connectreq: HttpStream): reqheaders = connectreq.headers() # if ":" in connectreq.path(): parsed_url = connectreq.url().split('//')[1].split(':') if len(parsed_url) > 1: host = parsed_url[0] port = parsed_url[1].split('/')[0] else: host, port = reqheaders['host'], 80 if 'Proxy-Connection' in reqheaders: del reqheaders['Proxy-Connection'] elif 'proxy-connection' in reqheaders: del reqheaders['proxy-connection'] # request to mail.ru as client sock_server = socket.create_connection((host, port)) try: clientreq = join_data(connectreq, 'Request') except (http_parser.http.ParserError, http_parser.http.BadStatusLine, ConnectionResetError): return sock_server.sendall(clientreq) # request from mail.ru to localhost as client if connectreq.method() == 'HEAD': return server_reader = SocketReader(sock_server) server_resp = HttpStream(server_reader, decompress=True) try: server_resp_tosend = join_data(server_resp, 'Response') except (http_parser.http.ParserError, http_parser.http.BadStatusLine, ConnectionResetError): return self.request.sendall(server_resp_tosend) req = Request(connectreq.method(), server_resp.status_code(), clientreq, host) self.add_date_base(connectreq, req)
def send_downstream(self, p, rhost, rport, method, url, upsock): with self.connect_downstream(rhost, rport) as downsock: request = '{method} {url} HTTP/{version[0]}.{version[1]}\r\n{headers}\r\n\r\n'.format(method=method.decode(), url=url, version=p.version(), headers='\r\n'.join(['{0}: {1}'.format(name, value) for name, value in p.headers().items()])) self.logger.debug('sending request {0!r}'.format(request)) downsock.send(request.encode()) downstream = HttpStream(SocketReader(downsock)) self.logger.debug('response: header={0}'.format(downstream.headers())) upsock.send('HTTP/{version[0]}.{version[1]} {code} {status}\r\n{headers}\r\n\r\n'.format(version=downstream.version(), code=downstream.status_code(), status=downstream.status(), headers='\r\n'.join(['{0}: {1}'.format(name, value) for name, value in downstream.headers().items()])).encode()) upsock.send(downstream.body_string())
def test_204_no_content(): stream = io.BytesIO(b'HTTP/1.1 204 No Content\r\n\r\n') response = HttpStream(stream) assert response.status_code() == 204 assert response.body_string() == b''