class JobClient(protocol.Protocol): def __init__(self, factory, params): self.parser = HttpParser() self.params = params self.verb = self.params.get_verb() self.headers = self.params.get_headers() self.uris = {} self.uris["getgame"] = "%s/game" % prefix self.uris["gethost"] = "%s/host" % prefix self.uris["getservice"] = "%s/service" % prefix self.recv = "" self.request = "" self.payload = None def no_unicode(self, text): if isinstance(text, unicode): return text.encode('utf-8') else: return text def check_json(self): try: return json.loads(self.recv) except: return False def TimedOut(self): pass def connectionMade(self): if self.verb == "GET": self.request = "GET %s HTTP/1.1\r\n%s\r\n" % (self.url, self.headers) elif self.verb == "POST": self.payload = self.params.get_payload() self.request = "POST %s HTTP/1.1\r\n%s\r\n%s" % \ (self.url, self.headers, self.payload) self.transport.write(self.request) def dataReceived(self, data): self.parser.execute(data, len(data)) if self.parser.is_headers_complete(): self.headers = self.parser.get_headers() if self.parser.is_partial_body(): self.recv += self.parser.recv_body() if self.parser.is_message_complete(): if self.check_json(): self.proc_response() else: print "Problem with %s" % self.recv def proc_response(self): #Override in subclass pass
def repeat(req_id): sqlite_con = saver.get_connection() cursor = sqlite_con.cursor() _, host, port, request, is_https = saver.get_request(cursor, req_id) sqlite_con.close() # Connecting to server sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect((host, port)) if is_https: context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) sock = context.wrap_socket(sock, server_hostname=host) sock.send(request) # Getting response parser = HttpParser() resp = b'' while True: data = sock.recv(buffer_size) if not data: break received = len(data) _ = parser.execute(data, received) if parser.is_partial_body(): resp += parser.recv_body() if parser.is_message_complete(): break headers = parser.get_headers() # Decode answer if headers['CONTENT-ENCODING'] == 'gzip': resp = gzip.decompress(resp) resp = str(resp, 'utf-8') else: try: resp = resp.decode('utf-8') except UnicodeDecodeError: print('Body wasn\'t decoded') print("{} HTTP/{}.{}".format(parser.get_status_code(), *parser.get_version())) for header in headers: print('{}: {}'.format(header, headers.get(header))) print() print(resp) print()
def fetch( url, method="GET", headers=None, body=None, connect_timeout=DEFAULT_CONNECT_TIMEOUT, request_timeout=DEFAULT_REQUEST_TIMEOUT, io_loop=None, resolver=resolve, max_buffer_size=DEFAULT_BUFFER_SIZE, follow_redirects=False, max_redirects=DEFAULT_MAX_REDIRECTS, validate_cert=config.http_client.validate_certs, allow_proxy=False, proxies=None, user=None, password=None, content_encoding=None, eof_mark=None, ): """ :param url: Fetch URL :param method: request method "GET", "POST", "PUT" etc :param headers: Dict of additional headers :param body: Request body for POST and PUT request :param connect_timeout: :param request_timeout: :param io_loop: :param resolver: :param follow_redirects: :param max_redirects: :param validate_cert: :param allow_proxy: :param proxies: :param user: :param password: :param max_buffer_size: :param content_encoding: :param eof_mark: Do not consider connection reset as error if eof_mark received (string or list) :return: code, headers, body """ def get_ssl_options(): ssl_options = {} if validate_cert: ssl_options["cert_reqs"] = ssl.CERT_REQUIRED return ssl_options logger.debug("HTTP %s %s", method, url) metrics["httpclient_requests", ("method", method.lower())] += 1 # Detect proxy when necessary io_loop = io_loop or tornado.ioloop.IOLoop.current() u = urlparse(str(url)) use_tls = u.scheme == "https" if ":" in u.netloc: host, port = u.netloc.rsplit(":") port = int(port) else: host = u.netloc port = DEFAULT_PORTS.get(u.scheme) if not port: raise tornado.gen.Return( (ERR_TIMEOUT, {}, "Cannot resolve port for scheme: %s" % u.scheme)) if is_ipv4(host): addr = host else: addr = yield resolver(host) if not addr: raise tornado.gen.Return( (ERR_TIMEOUT, {}, "Cannot resolve host: %s" % host)) # Detect proxy server if allow_proxy: proxy = (proxies or SYSTEM_PROXIES).get(u.scheme) else: proxy = None # Connect stream = None s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: if use_tls and not proxy: stream = tornado.iostream.SSLIOStream( s, io_loop=io_loop, ssl_options=get_ssl_options()) else: stream = tornado.iostream.IOStream(s, io_loop=io_loop) try: if proxy: connect_address = proxy elif isinstance(addr, tuple): connect_address = addr else: connect_address = (addr, port) if proxy: logger.debug("Connecting to proxy %s:%s", connect_address[0], connect_address[1]) yield tornado.gen.with_timeout( io_loop.time() + connect_timeout, future=stream.connect(connect_address, server_hostname=u.netloc), io_loop=io_loop, ) except tornado.iostream.StreamClosedError: metrics["httpclient_timeouts"] += 1 raise tornado.gen.Return((ERR_TIMEOUT, {}, "Connection refused")) except tornado.gen.TimeoutError: metrics["httpclient_timeouts"] += 1 raise tornado.gen.Return((ERR_TIMEOUT, {}, "Connection timed out")) deadline = io_loop.time() + request_timeout # Proxy CONNECT if proxy: logger.debug("Sending CONNECT %s:%s", addr, port) # Send CONNECT request req = b"CONNECT %s:%s HTTP/1.1\r\nUser-Agent: %s\r\n\r\n" % ( addr, port, DEFAULT_USER_AGENT, ) try: yield tornado.gen.with_timeout( deadline, future=stream.write(req), io_loop=io_loop, quiet_exceptions=(tornado.iostream.StreamClosedError, ), ) except tornado.iostream.StreamClosedError: metrics["httpclient_proxy_timeouts"] += 1 raise tornado.gen.Return( (ERR_TIMEOUT, {}, "Connection reset while connecting to proxy")) except tornado.gen.TimeoutError: metrics["httpclient_proxy_timeouts"] += 1 raise tornado.gen.Return( (ERR_TIMEOUT, {}, "Timed out while sending request to proxy")) # Wait for proxy response parser = HttpParser() while not parser.is_headers_complete(): try: data = yield tornado.gen.with_timeout( deadline, future=stream.read_bytes(max_buffer_size, partial=True), io_loop=io_loop, quiet_exceptions=( tornado.iostream.StreamClosedError, ), ) except tornado.iostream.StreamClosedError: metrics["httpclient_proxy_timeouts"] += 1 raise tornado.gen.Return( (ERR_TIMEOUT, {}, "Connection reset while connecting to proxy")) except tornado.gen.TimeoutError: metrics["httpclient_proxy_timeouts"] += 1 raise tornado.gen.Return( (ERR_TIMEOUT, {}, "Timed out while sending request to proxy")) received = len(data) parsed = parser.execute(data, received) if parsed != received: raise tornado.gen.Return( (ERR_PARSE_ERROR, {}, "Parse error")) code = parser.get_status_code() logger.debug("Proxy response: %s", code) if not 200 <= code <= 299: raise tornado.gen.Return( (code, parser.get_headers(), "Proxy error: %s" % code)) # Switch to TLS when necessary if use_tls: logger.debug("Starting TLS negotiation") try: stream = yield tornado.gen.with_timeout( deadline, future=stream.start_tls( server_side=False, ssl_options=get_ssl_options(), server_hostname=u.netloc, ), io_loop=io_loop, quiet_exceptions=( tornado.iostream.StreamClosedError, ), ) except tornado.iostream.StreamClosedError: metrics["httpclient_proxy_timeouts"] += 1 raise tornado.gen.Return( (ERR_TIMEOUT, {}, "Connection reset while connecting to proxy")) except tornado.gen.TimeoutError: metrics["httpclient_proxy_timeouts"] += 1 raise tornado.gen.Return( (ERR_TIMEOUT, {}, "Timed out while sending request to proxy")) # Process request body = body or "" content_type = "application/binary" if isinstance(body, unicode): body = body.encode("utf-8") elif not isinstance(body, six.string_types): body = ujson.dumps(body) content_type = "text/json" h = { "Host": str(u.netloc), "Connection": "close", "User-Agent": DEFAULT_USER_AGENT } if body and content_encoding: if content_encoding == CE_DEFLATE: # Deflate compression h["Content-Encoding"] = CE_DEFLATE compress = zlib.compressobj( zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -zlib.MAX_WBITS, zlib.DEF_MEM_LEVEL, zlib.Z_DEFAULT_STRATEGY, ) body = compress.compress(body) + compress.flush() elif content_encoding == CE_GZIP: # gzip compression h["Content-Encoding"] = CE_GZIP compress = zlib.compressobj(6, zlib.DEFLATED, -zlib.MAX_WBITS, zlib.DEF_MEM_LEVEL, 0) crc = zlib.crc32(body, 0) & 0xFFFFFFFF body = "\x1f\x8b\x08\x00%s\x02\xff%s%s%s%s" % ( to32u(int(time.time())), compress.compress(body), compress.flush(), to32u(crc), to32u(len(body)), ) if method in REQUIRE_LENGTH_METHODS: h["Content-Length"] = str(len(body)) h["Content-Type"] = content_type if user and password: # Include basic auth header h["Authorization"] = "Basic %s" % ( "%s:%s" % (user, password)).encode("base64").strip() if headers: h.update(headers) path = u.path if u.query: path += "?%s" % u.query req = b"%s %s HTTP/1.1\r\n%s\r\n\r\n%s" % ( method, path, "\r\n".join(b"%s: %s" % (k, h[k]) for k in h), body, ) try: yield tornado.gen.with_timeout( deadline, future=stream.write(req), io_loop=io_loop, quiet_exceptions=(tornado.iostream.StreamClosedError, ), ) except tornado.iostream.StreamClosedError: metrics["httpclient_timeouts"] += 1 raise tornado.gen.Return( (ERR_TIMEOUT, {}, "Connection reset while sending request")) except tornado.gen.TimeoutError: metrics["httpclient_timeouts"] += 1 raise tornado.gen.Return( (ERR_TIMEOUT, {}, "Timed out while sending request")) parser = HttpParser() response_body = [] while not parser.is_message_complete(): try: data = yield tornado.gen.with_timeout( deadline, future=stream.read_bytes(max_buffer_size, partial=True), io_loop=io_loop, quiet_exceptions=(tornado.iostream.StreamClosedError, ), ) except tornado.iostream.StreamClosedError: if not response_body and config.features.pypy: break if eof_mark and response_body: # Check if EOF mark is in received data response_body = ["".join(response_body)] if isinstance(eof_mark, six.string_types): if eof_mark in response_body[0]: break else: found = False for m in eof_mark: if m in response_body[0]: found = True break if found: break metrics["httpclient_timeouts"] += 1 raise tornado.gen.Return( (ERR_READ_TIMEOUT, {}, "Connection reset")) except tornado.gen.TimeoutError: metrics["httpclient_timeouts"] += 1 raise tornado.gen.Return( (ERR_READ_TIMEOUT, {}, "Request timed out")) received = len(data) parsed = parser.execute(data, received) if parsed != received: raise tornado.gen.Return((ERR_PARSE_ERROR, {}, "Parse error")) if parser.is_partial_body(): response_body += [parser.recv_body()] code = parser.get_status_code() parsed_headers = parser.get_headers() logger.debug("HTTP Response %s", code) if 300 <= code <= 399 and follow_redirects: # Process redirects if max_redirects > 0: new_url = parsed_headers.get("Location") if not new_url: raise tornado.gen.Return( (ERR_PARSE_ERROR, {}, "No Location header")) logger.debug("HTTP redirect %s %s", code, new_url) code, parsed_headers, response_body = yield fetch( new_url, method="GET", headers=headers, connect_timeout=connect_timeout, request_timeout=request_timeout, resolver=resolver, max_buffer_size=max_buffer_size, follow_redirects=follow_redirects, max_redirects=max_redirects - 1, validate_cert=validate_cert, allow_proxy=allow_proxy, proxies=proxies, ) raise tornado.gen.Return((code, parsed_headers, response_body)) else: raise tornado.gen.Return((404, {}, "Redirect limit exceeded")) # @todo: Process gzip and deflate Content-Encoding raise tornado.gen.Return( (code, parsed_headers, "".join(response_body))) finally: if stream: stream.close() else: s.close()