def check_correct_GET(host, port, version="1.1"): #Check if only one response from each GET error_reported = "No error reported, check number of responses to confirm" s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) responses = 0 status = 0 buf_size = 1024 data_all = "" try: s.connect((host, port)) s.send("GET /index.html HTTP/" + version + "\r\nHost: %s:%d\ \r\nConnection:Keep-Alive\r\n\r\n" % (host, port)) while True: s.settimeout(1) data = s.recv(buf_size) data_all += data if not data: break except socket.timeout: pass finally: s.close() p = HttpParser() while len(data_all) > 0: nparsed = p.execute(data_all, len(data_all)) if nparsed == 0: break status = p.get_status_code() if p.is_message_complete(): responses += 1 if nparsed < len(data_all): responses += 1 #more data if p.get_status_code() >= 400: error_reported = "Error found" p = HttpParser() # create another data_all = data_all[nparsed:] return error_reported, responses, status
def repeat(req_id): sqlite_con = saver.get_connection() cursor = sqlite_con.cursor() _, host, port, request, is_https = saver.get_request(cursor, req_id) sqlite_con.close() # Connecting to server sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect((host, port)) if is_https: context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) sock = context.wrap_socket(sock, server_hostname=host) sock.send(request) # Getting response parser = HttpParser() resp = b'' while True: data = sock.recv(buffer_size) if not data: break received = len(data) _ = parser.execute(data, received) if parser.is_partial_body(): resp += parser.recv_body() if parser.is_message_complete(): break headers = parser.get_headers() # Decode answer if headers['CONTENT-ENCODING'] == 'gzip': resp = gzip.decompress(resp) resp = str(resp, 'utf-8') else: try: resp = resp.decode('utf-8') except UnicodeDecodeError: print('Body wasn\'t decoded') print("{} HTTP/{}.{}".format(parser.get_status_code(), *parser.get_version())) for header in headers: print('{}: {}'.format(header, headers.get(header))) print() print(resp) print()
def check_correct_GET(host, port): #Check if only one response from each GET error_reported = False s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) responses = 0 buf_size = 1024 data_all = "" try: s.connect((host, port)) s.send("GET /index.html HTTP/1.1\r\nHost: %s:%d\ \r\nConnection:Keep-Alive\r\n\r\n" % (host, port)) while True: s.settimeout(1) data = s.recv(buf_size) data_all += data if not data: break except socket.timeout: pass finally: s.close() p = HttpParser() while len(data_all) > 0: nparsed = p.execute(data_all, len(data_all)) if nparsed == 0: break if p.is_message_complete(): responses += 1 if nparsed < len(data_all): responses += 1 #more data if p.get_status_code() >= 400: error_reported = True p = HttpParser() # create another data_all = data_all[nparsed:] return error_reported, responses
def fetch( url, method="GET", headers=None, body=None, connect_timeout=DEFAULT_CONNECT_TIMEOUT, request_timeout=DEFAULT_REQUEST_TIMEOUT, io_loop=None, resolver=resolve, max_buffer_size=DEFAULT_BUFFER_SIZE, follow_redirects=False, max_redirects=DEFAULT_MAX_REDIRECTS, validate_cert=config.http_client.validate_certs, allow_proxy=False, proxies=None, user=None, password=None, content_encoding=None, eof_mark=None, ): """ :param url: Fetch URL :param method: request method "GET", "POST", "PUT" etc :param headers: Dict of additional headers :param body: Request body for POST and PUT request :param connect_timeout: :param request_timeout: :param io_loop: :param resolver: :param follow_redirects: :param max_redirects: :param validate_cert: :param allow_proxy: :param proxies: :param user: :param password: :param max_buffer_size: :param content_encoding: :param eof_mark: Do not consider connection reset as error if eof_mark received (string or list) :return: code, headers, body """ def get_ssl_options(): ssl_options = {} if validate_cert: ssl_options["cert_reqs"] = ssl.CERT_REQUIRED return ssl_options logger.debug("HTTP %s %s", method, url) metrics["httpclient_requests", ("method", method.lower())] += 1 # Detect proxy when necessary io_loop = io_loop or tornado.ioloop.IOLoop.current() u = urlparse(str(url)) use_tls = u.scheme == "https" if ":" in u.netloc: host, port = u.netloc.rsplit(":") port = int(port) else: host = u.netloc port = DEFAULT_PORTS.get(u.scheme) if not port: raise tornado.gen.Return( (ERR_TIMEOUT, {}, "Cannot resolve port for scheme: %s" % u.scheme)) if is_ipv4(host): addr = host else: addr = yield resolver(host) if not addr: raise tornado.gen.Return( (ERR_TIMEOUT, {}, "Cannot resolve host: %s" % host)) # Detect proxy server if allow_proxy: proxy = (proxies or SYSTEM_PROXIES).get(u.scheme) else: proxy = None # Connect stream = None s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: if use_tls and not proxy: stream = tornado.iostream.SSLIOStream( s, io_loop=io_loop, ssl_options=get_ssl_options()) else: stream = tornado.iostream.IOStream(s, io_loop=io_loop) try: if proxy: connect_address = proxy elif isinstance(addr, tuple): connect_address = addr else: connect_address = (addr, port) if proxy: logger.debug("Connecting to proxy %s:%s", connect_address[0], connect_address[1]) yield tornado.gen.with_timeout( io_loop.time() + connect_timeout, future=stream.connect(connect_address, server_hostname=u.netloc), io_loop=io_loop, ) except tornado.iostream.StreamClosedError: metrics["httpclient_timeouts"] += 1 raise tornado.gen.Return((ERR_TIMEOUT, {}, "Connection refused")) except tornado.gen.TimeoutError: metrics["httpclient_timeouts"] += 1 raise tornado.gen.Return((ERR_TIMEOUT, {}, "Connection timed out")) deadline = io_loop.time() + request_timeout # Proxy CONNECT if proxy: logger.debug("Sending CONNECT %s:%s", addr, port) # Send CONNECT request req = b"CONNECT %s:%s HTTP/1.1\r\nUser-Agent: %s\r\n\r\n" % ( addr, port, DEFAULT_USER_AGENT, ) try: yield tornado.gen.with_timeout( deadline, future=stream.write(req), io_loop=io_loop, quiet_exceptions=(tornado.iostream.StreamClosedError, ), ) except tornado.iostream.StreamClosedError: metrics["httpclient_proxy_timeouts"] += 1 raise tornado.gen.Return( (ERR_TIMEOUT, {}, "Connection reset while connecting to proxy")) except tornado.gen.TimeoutError: metrics["httpclient_proxy_timeouts"] += 1 raise tornado.gen.Return( (ERR_TIMEOUT, {}, "Timed out while sending request to proxy")) # Wait for proxy response parser = HttpParser() while not parser.is_headers_complete(): try: data = yield tornado.gen.with_timeout( deadline, future=stream.read_bytes(max_buffer_size, partial=True), io_loop=io_loop, quiet_exceptions=( tornado.iostream.StreamClosedError, ), ) except tornado.iostream.StreamClosedError: metrics["httpclient_proxy_timeouts"] += 1 raise tornado.gen.Return( (ERR_TIMEOUT, {}, "Connection reset while connecting to proxy")) except tornado.gen.TimeoutError: metrics["httpclient_proxy_timeouts"] += 1 raise tornado.gen.Return( (ERR_TIMEOUT, {}, "Timed out while sending request to proxy")) received = len(data) parsed = parser.execute(data, received) if parsed != received: raise tornado.gen.Return( (ERR_PARSE_ERROR, {}, "Parse error")) code = parser.get_status_code() logger.debug("Proxy response: %s", code) if not 200 <= code <= 299: raise tornado.gen.Return( (code, parser.get_headers(), "Proxy error: %s" % code)) # Switch to TLS when necessary if use_tls: logger.debug("Starting TLS negotiation") try: stream = yield tornado.gen.with_timeout( deadline, future=stream.start_tls( server_side=False, ssl_options=get_ssl_options(), server_hostname=u.netloc, ), io_loop=io_loop, quiet_exceptions=( tornado.iostream.StreamClosedError, ), ) except tornado.iostream.StreamClosedError: metrics["httpclient_proxy_timeouts"] += 1 raise tornado.gen.Return( (ERR_TIMEOUT, {}, "Connection reset while connecting to proxy")) except tornado.gen.TimeoutError: metrics["httpclient_proxy_timeouts"] += 1 raise tornado.gen.Return( (ERR_TIMEOUT, {}, "Timed out while sending request to proxy")) # Process request body = body or "" content_type = "application/binary" if isinstance(body, unicode): body = body.encode("utf-8") elif not isinstance(body, six.string_types): body = ujson.dumps(body) content_type = "text/json" h = { "Host": str(u.netloc), "Connection": "close", "User-Agent": DEFAULT_USER_AGENT } if body and content_encoding: if content_encoding == CE_DEFLATE: # Deflate compression h["Content-Encoding"] = CE_DEFLATE compress = zlib.compressobj( zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -zlib.MAX_WBITS, zlib.DEF_MEM_LEVEL, zlib.Z_DEFAULT_STRATEGY, ) body = compress.compress(body) + compress.flush() elif content_encoding == CE_GZIP: # gzip compression h["Content-Encoding"] = CE_GZIP compress = zlib.compressobj(6, zlib.DEFLATED, -zlib.MAX_WBITS, zlib.DEF_MEM_LEVEL, 0) crc = zlib.crc32(body, 0) & 0xFFFFFFFF body = "\x1f\x8b\x08\x00%s\x02\xff%s%s%s%s" % ( to32u(int(time.time())), compress.compress(body), compress.flush(), to32u(crc), to32u(len(body)), ) if method in REQUIRE_LENGTH_METHODS: h["Content-Length"] = str(len(body)) h["Content-Type"] = content_type if user and password: # Include basic auth header h["Authorization"] = "Basic %s" % ( "%s:%s" % (user, password)).encode("base64").strip() if headers: h.update(headers) path = u.path if u.query: path += "?%s" % u.query req = b"%s %s HTTP/1.1\r\n%s\r\n\r\n%s" % ( method, path, "\r\n".join(b"%s: %s" % (k, h[k]) for k in h), body, ) try: yield tornado.gen.with_timeout( deadline, future=stream.write(req), io_loop=io_loop, quiet_exceptions=(tornado.iostream.StreamClosedError, ), ) except tornado.iostream.StreamClosedError: metrics["httpclient_timeouts"] += 1 raise tornado.gen.Return( (ERR_TIMEOUT, {}, "Connection reset while sending request")) except tornado.gen.TimeoutError: metrics["httpclient_timeouts"] += 1 raise tornado.gen.Return( (ERR_TIMEOUT, {}, "Timed out while sending request")) parser = HttpParser() response_body = [] while not parser.is_message_complete(): try: data = yield tornado.gen.with_timeout( deadline, future=stream.read_bytes(max_buffer_size, partial=True), io_loop=io_loop, quiet_exceptions=(tornado.iostream.StreamClosedError, ), ) except tornado.iostream.StreamClosedError: if not response_body and config.features.pypy: break if eof_mark and response_body: # Check if EOF mark is in received data response_body = ["".join(response_body)] if isinstance(eof_mark, six.string_types): if eof_mark in response_body[0]: break else: found = False for m in eof_mark: if m in response_body[0]: found = True break if found: break metrics["httpclient_timeouts"] += 1 raise tornado.gen.Return( (ERR_READ_TIMEOUT, {}, "Connection reset")) except tornado.gen.TimeoutError: metrics["httpclient_timeouts"] += 1 raise tornado.gen.Return( (ERR_READ_TIMEOUT, {}, "Request timed out")) received = len(data) parsed = parser.execute(data, received) if parsed != received: raise tornado.gen.Return((ERR_PARSE_ERROR, {}, "Parse error")) if parser.is_partial_body(): response_body += [parser.recv_body()] code = parser.get_status_code() parsed_headers = parser.get_headers() logger.debug("HTTP Response %s", code) if 300 <= code <= 399 and follow_redirects: # Process redirects if max_redirects > 0: new_url = parsed_headers.get("Location") if not new_url: raise tornado.gen.Return( (ERR_PARSE_ERROR, {}, "No Location header")) logger.debug("HTTP redirect %s %s", code, new_url) code, parsed_headers, response_body = yield fetch( new_url, method="GET", headers=headers, connect_timeout=connect_timeout, request_timeout=request_timeout, resolver=resolver, max_buffer_size=max_buffer_size, follow_redirects=follow_redirects, max_redirects=max_redirects - 1, validate_cert=validate_cert, allow_proxy=allow_proxy, proxies=proxies, ) raise tornado.gen.Return((code, parsed_headers, response_body)) else: raise tornado.gen.Return((404, {}, "Redirect limit exceeded")) # @todo: Process gzip and deflate Content-Encoding raise tornado.gen.Return( (code, parsed_headers, "".join(response_body))) finally: if stream: stream.close() else: s.close()
class HttpProtocolHandler: PROXY_TUNNEL_ESTABLISHED_RESPONSE_PKT = build_http_response( 200, reason=b'Connection established') def __init__(self, client_conn: socket.socket, client_addr, flags, req): self.start_time: float = time.time() self.client = client_conn self.req = req self.client_addr = client_addr # host and socket_fd self.flags = flags self.request_parser = HttpParser(0) # 0 - parse only requests self.response_parser = HttpParser(1) # 1 - parse only responses self.total_response_size: int = 0 self.upstream: Optional[urlparse.SplitResultBytes] = None self.host = None self.port = None self.upstream_url = None self.server: Optional[socket.socket] = None def parse_url(self, parser): url = parser.get_url() method = parser.get_method() protocol_pos = url.find('://') if protocol_pos != -1: url = url[(protocol_pos + 3):] port_pos = url.find(':') host_pos = url.find('/') if host_pos == -1: host_pos = len(url) if port_pos == -1 or host_pos < port_pos: port = 443 if method == "CONNECT" else DEFAULT_HTTP_PORT else: port = int((url[port_pos + 1:])[:host_pos - port_pos - 1]) port_ind = url.find(':') if port_ind != -1: url = url[:port_ind] self.upstream = urlparse.urlsplit('http://' + url + '/') self.upstream_url = self.build_upstream_relative_path() host = self.upstream.hostname port_ind = host.find(':') if port_ind != -1: host = host[:port_ind] return host, port def run(self) -> None: p = HttpParser() try: p.execute(self.req, len(self.req)) url = p.get_url() metopd = p.get_method() http_pos = url.find('://') if http_pos == -1: temp = url else: temp = url[(http_pos + 3):] port_pos = temp.find(':') host_pos = temp.find('/') if host_pos == -1: host_pos = len(temp) if port_pos == -1 or host_pos < port_pos: port = 443 if metopd == "CONNECT" else 80 else: port = int((temp[port_pos + 1:])[:host_pos - port_pos - 1]) host = p.get_headers()['host'] port_ind = host.find(':') if port_ind != -1: host = host[:port_ind] if metopd == "CONNECT": https_proxy(host, port, self.client) else: proxy(host, port, self.client, self.req) except Exception as e: print(e) pass def access_log(self): server_host, server_port = self.upstream.hostname, self.upstream.port \ if self.upstream.port else DEFAULT_HTTP_PORT connection_time_ms = (time.time() - self.start_time) * 1000 method = self.request_parser.get_method() if method == httpMethods.CONNECT: pass elif method: print('pid:%s | %s:%s - %s %s:%s%s - %s %s - %s bytes - %.2f ms' % (str(getpid()), self.client_addr[0], self.client_addr[1], method, server_host, server_port, self.request_parser.get_path(), self.response_parser.get_status_code(), self.response_parser.get_errno(), self.total_response_size, connection_time_ms))