def report_connect_fail(self, ip, force_remove=False): self.ip_lock.acquire() try: time_now = time.time() if not ip in self.ip_dict: xlog.debug("report_connect_fail %s not exist", ip) return if force_remove: if self.ip_dict[ip]['fail_times'] == 0: self.good_ip_num -= 1 self.bad_ip_num += 1 del self.ip_dict[ip] if ip in self.gws_ip_list: self.gws_ip_list.remove(ip) xlog.info("remove ip:%s left amount:%d gws_num:%d", ip, len(self.ip_dict), len(self.gws_ip_list)) return self.ip_dict[ip]['links'] -= 1 # ignore if system network is disconnected. if not check_local_network.is_ok(): xlog.debug("report_connect_fail network fail") return check_local_network.report_network_fail() if not check_local_network.is_ok(): return fail_time = self.ip_dict[ip]["fail_time"] if time_now - fail_time < 1: xlog.debug("fail time too near %s", ip) return if self.ip_dict[ip]['fail_times'] == 0: self.good_ip_num -= 1 self.bad_ip_num += 1 self.ip_dict[ip]['fail_times'] += 1 self.append_ip_history(ip, "fail") self.ip_dict[ip]["fail_time"] = time_now self.to_check_ip_queue.put((ip, time_now + 10)) xlog.debug("report_connect_fail:%s", ip) except Exception as e: xlog.exception("report_connect_fail err:%s", e) finally: self.iplist_need_save = True self.ip_lock.release() if not self.is_ip_enough(): self.search_more_google_ip()
def check_ip_process(self): while connect_control.keep_running: try: ip, test_time = self.to_check_ip_queue.get() except: continue time_wait = test_time - time.time() if time_wait > 0: time.sleep(time_wait) if not check_local_network.is_ok(): try: if self.ip_dict[ip]['fail_times']: self.ip_dict[ip]['fail_times'] = 0 self.good_ip_num += 1 except: pass continue result = check_ip.test_xtunnel_ip2(ip) if result and result.support_xtunnel: self.add_ip(ip, result.request_time, result.domain, "gws") xlog.debug("restore ip:%s", ip) continue xlog.debug("ip:%s real fail", ip)
def check_ip_process(self): while connect_control.keep_running: try: ip, test_time = self.to_check_ip_queue.get() except: continue time_wait = test_time - time.time() if time_wait > 0: time.sleep(time_wait) if not check_local_network.is_ok(): try: if self.ip_dict[ip]['fail_times']: self.ip_dict[ip]['fail_times'] = 0 self.good_ip_num += 1 self.bad_ip_num -= 1 except: pass continue result = check_ip.test_gae_ip2(ip) if result and result.support_gae: self.add_ip(ip, result.handshake_time, result.domain, "gws") xlog.debug("restore ip:%s", ip) continue xlog.debug("ip:%s real fail", ip)
def do_METHOD(self): self.req_payload = None host = self.headers.get('Host', '') host_ip, _, port = host.rpartition(':') if self.is_local([host, host_ip]): logger.debug("Browse localhost by proxy") return self.forward_local() elif host == self.fake_host: # logger.debug("%s %s", self.command, self.path) # for web_ui status page # auto detect browser proxy setting is work return self.wfile.write(self.self_check_response_data) if not (front.config.use_ipv6 == "force_ipv6" and \ check_local_network.IPv6.is_ok() or \ front.config.use_ipv6 != "force_ipv6" and \ check_local_network.is_ok()): self.close_connection = 1 return if isinstance(self.connection, ssl.SSLSocket): schema = "https" else: schema = "http" if self.path[0] == '/': self.host = self.headers['Host'] self.url = '%s://%s%s' % (schema, host, self.path) else: self.url = self.path self.parsed_url = urlparse.urlparse(self.path) self.host = self.parsed_url[1] if len(self.parsed_url[4]): self.path = '?'.join([self.parsed_url[2], self.parsed_url[4]]) else: self.path = self.parsed_url[2] if len(self.url) > 2083 and self.host.endswith( front.config.GOOGLE_ENDSWITH): return self.go_DIRECT() if self.host in front.config.HOSTS_GAE: return self.go_AGENT() # redirect http request to https request # avoid key word filter when pass through GFW if host in front.config.HOSTS_DIRECT: return self.go_DIRECT() if host.endswith(front.config.HOSTS_GAE_ENDSWITH): return self.go_AGENT() if host.endswith(front.config.HOSTS_DIRECT_ENDSWITH): return self.go_DIRECT() return self.go_AGENT()
def do_METHOD(self): self.req_payload = None host = self.headers.get('Host', '') host_ip, _, port = host.rpartition(':') if self.is_local([host, host_ip]): xlog.debug("Browse localhost by proxy") return self.forward_local() elif host == self.fake_host: # xlog.debug("%s %s", self.command, self.path) # for web_ui status page # auto detect browser proxy setting is work return self.wfile.write(self.self_check_response_data) if not (front.config.use_ipv6 == "force_ipv6" and \ check_local_network.IPv6.is_ok() or \ front.config.use_ipv6 != "force_ipv6" and \ check_local_network.is_ok()): self.close_connection = 1 return if isinstance(self.connection, ssl.SSLSocket): schema = "https" else: schema = "http" if self.path[0] == '/': self.host = self.headers['Host'] self.url = '%s://%s%s' % (schema, host, self.path) else: self.url = self.path self.parsed_url = urlparse.urlparse(self.path) self.host = self.parsed_url[1] if len(self.parsed_url[4]): self.path = '?'.join([self.parsed_url[2], self.parsed_url[4]]) else: self.path = self.parsed_url[2] if len(self.url) > 2083 and self.host.endswith(front.config.GOOGLE_ENDSWITH): return self.go_DIRECT() if self.host in front.config.HOSTS_GAE: return self.go_AGENT() # redirect http request to https request # avoid key word filter when pass through GFW if host in front.config.HOSTS_DIRECT: return self.go_DIRECT() if host.endswith(front.config.HOSTS_GAE_ENDSWITH): return self.go_AGENT() if host.endswith(front.config.HOSTS_DIRECT_ENDSWITH): return self.go_DIRECT() return self.go_AGENT()
def _create_ssl_connection(self, ip_port): if not connect_control.allow_connect(): time.sleep(10) return False ip = ip_port[0] port = ip_port[1] connect_control.start_connect_register(high_prior=True) time_begin = time.time() try: xlog.debug("try connect %s", ip) ssl_sock = check_ip.connect_ssl(ip, port=port, timeout=self.connect_timeout, on_close=ip_manager.ssl_closed) ip_manager.update_ip(ip, ssl_sock.handshake_time) xlog.debug("create_ssl update ip:%s time:%d h2:%d sni:%s top:%s", ip, ssl_sock.handshake_time, ssl_sock.h2, ssl_sock.sni, ssl_sock.top_domain) ssl_sock.last_use_time = ssl_sock.create_time ssl_sock.received_size = 0 ssl_sock.load = 0 ssl_sock.host = self.sub + "." + ssl_sock.top_domain connect_control.report_connect_success() return ssl_sock except Exception as e: time_cost = time.time() - time_begin if time_cost < self.connect_timeout - 1: xlog.debug("connect %s fail:%s cost:%d ", ip, e, time_cost * 1000) else: xlog.debug("%s fail:%r", ip, e) ip_manager.report_connect_fail(ip) connect_control.report_connect_fail() if not check_local_network.is_ok(): time.sleep(10) else: time.sleep(1) return False finally: connect_control.end_connect_register(high_prior=True)
def recheck_ip(self, ip): # recheck ip if not work. # can block. if not check_local_network.is_ok(): xlog.debug("recheck_ip:%s network is fail", ip) return self.report_connect_fail(ip) connect_control.start_connect_register() result = check_ip.test_gae_ip2(ip) connect_control.end_connect_register() if not result: self.report_connect_fail(ip, force_remove=True) xlog.debug("recheck_ip:%s real fail, removed.", ip) else: self.add_ip(ip, result.handshake_time, result.domain, "gws") xlog.debug("recheck_ip:%s restore okl", ip)
def recheck_ip(self, ip): # recheck ip if not work. # can block. if not check_local_network.is_ok(): xlog.debug("recheck_ip:%s network is fail", ip) return self.report_connect_fail(ip) result = check_ip.test_gae_ip2(ip) if not result: # connect fail. # do nothing return if not result.support_gae: self.report_connect_fail(ip, force_remove=True) xlog.debug("recheck_ip:%s real fail, removed.", ip) else: self.add_ip(ip, result.handshake_time, result.domain, "gws") xlog.debug("recheck_ip:%s restore okl", ip)
def recheck_ip(self, ip): # recheck ip if not work. # can block. if not check_local_network.is_ok(): xlog.debug("recheck_ip:%s network is fail", ip) return self.report_connect_fail(ip) connect_control.start_connect_register() result = check_ip.test_xtunnel_ip2(ip) connect_control.end_connect_register() if not result: # connect fail. # do nothing return if not result.support_xtunnel: self.report_connect_fail(ip, force_remove=True) xlog.debug("recheck_ip:%s real fail, removed.", ip) else: self.add_ip(ip, result.request_time, result.domain, "gws") xlog.debug("recheck_ip:%s restore okl", ip)
def run(self): req_range_begin = None req_range_end = None for k, v in self.headers.items(): # xlog.debug("range req head:%s => %s", k, v) if k.lower() == "range": req_range_begin, req_range_end = tuple( x for x in re.search(r'bytes=(\d*)-(\d*)', v).group(1, 2)) # break response_headers = dict((k.title(), v) for k, v in self.response.headers.items()) content_range = response_headers['Content-Range'] res_begin, res_end, res_length = tuple(int(x) for x in re.search( r'bytes (\d+)-(\d+)/(\d+)', content_range).group(1, 2, 3)) self.req_begin = res_end + 1 if req_range_begin and req_range_end: self.req_end = int(req_range_end) else: self.req_end = res_length - 1 self.wait_begin = res_begin if self.wait_begin == 0 and self.req_end == res_length - 1: response_headers['Content-Length'] = str(res_length) del response_headers['Content-Range'] state_code = 200 else: response_headers['Content-Range'] = 'bytes %s-%s/%s' % ( res_begin, self.req_end, res_length) response_headers['Content-Length'] = str( self.req_end - res_begin + 1) state_code = 206 response_headers["Persist"] = "" response_headers["Connection"] = "Persist" xlog.info('RangeFetch %d-%d started(%r) ', res_begin, self.req_end, self.url) try: self.wfile.write("HTTP/1.1 %d OK\r\n" % state_code) for key in response_headers: if key in skip_response_headers: continue value = response_headers[key] #xlog.debug("Head %s: %s", key.title(), value) send_header(self.wfile, key, value) self.wfile.write("\r\n") except Exception as e: self.keep_running = False xlog.info("RangeFetch send response fail:%r %s", e, self.url) return data_left_to_fetch = self.req_end - self.req_begin + 1 fetch_times = int( (data_left_to_fetch + front.config.AUTORANGE_MAXSIZE - 1) / front.config.AUTORANGE_MAXSIZE) thread_num = min(front.config.AUTORANGE_THREADS, fetch_times) for i in xrange(0, thread_num): threading.Thread(target=self.fetch_worker).start() threading.Thread(target=self.fetch, args=( res_begin, res_end, self.response)).start() ok = "ok" while self.keep_running and \ (front.config.use_ipv6 == "force_ipv6" and \ check_local_network.IPv6.is_ok() or \ front.config.use_ipv6 != "force_ipv6" and \ check_local_network.is_ok()) and \ self.wait_begin < self.req_end + 1: with self.lock: if self.wait_begin not in self.data_list: self.waiter.wait() if self.wait_begin not in self.data_list: xlog.error("get notify but no data") continue else: data = self.data_list[self.wait_begin] del self.data_list[self.wait_begin] self.wait_begin += len(data) self.data_size -= len(data) self.all_data_size[self] = self.data_size try: ret = self.wfile.write(data) if ret == ssl.SSL_ERROR_WANT_WRITE or ret == ssl.SSL_ERROR_WANT_READ: xlog.debug( "send to browser wfile.write ret:%d, retry", ret) ret = self.wfile.write(data) xlog.debug("send to browser wfile.write ret:%d", ret) del data except Exception as e: xlog.info('RangeFetch client closed(%s). %s', e, self.url) ok = None break self.keep_running = False self.all_data_size.pop(self, None) return ok
def connect_ssl(ip, port=443, timeout=5, check_cert=True, close_cb=None): if not check_local_network.is_ok(ip): with network_fail_lock: time.sleep(0.1) ip_port = (ip, port) sni = sni_generater.get() if int(config.PROXY_ENABLE): sock = socks.socksocket(socket.AF_INET if ':' not in ip else socket.AF_INET6) else: sock = socket.socket(socket.AF_INET if ':' not in ip else socket.AF_INET6) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0)) # resize socket recv buffer 8K->32K to improve browser releated application performance sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32*1024) sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True) sock.settimeout(timeout) ssl_sock = openssl_wrap.SSLConnection(openssl_context, sock, ip, close_cb) ssl_sock.set_connect_state() if hasattr(ssl_sock, 'set_tlsext_host_name'): try: ssl_sock.set_tlsext_host_name(sni) except: pass time_begin = time.time() ssl_sock.connect(ip_port) time_connected = time.time() ssl_sock.do_handshake() try: h2 = ssl_sock.get_alpn_proto_negotiated() if h2 == "h2": ssl_sock.h2 = True else: ssl_sock.h2 = False xlog.debug("%s alpn h2:%s", ip, h2) except Exception as e: #xlog.exception("alpn:%r", e) if hasattr(ssl_sock._connection, "protos") and ssl_sock._connection.protos == "h2": ssl_sock.h2 = True # xlog.debug("ip:%s http/2", ip) else: ssl_sock.h2 = False # xlog.debug("ip:%s http/1.1", ip) time_handshaked = time.time() check_local_network.report_ok(ip) def verify_SSL_certificate_issuer(ssl_sock): # cert = ssl_sock.get_peer_certificate() # if not cert: # #google_ip.report_bad_ip(ssl_sock.ip) # #connect_control.fall_into_honeypot() # raise socket.error(' certficate is none') # issuer_commonname = next((v for k, v in cert.get_issuer().get_components() if k == 'CN'), '') # if not issuer_commonname.startswith('Google'): # google_ip.report_connect_fail(ip, force_remove=True) # raise socket.error(' certficate is issued by %r, not Google' % ( issuer_commonname)) certs = ssl_sock.get_peer_cert_chain() if not certs: # google_ip.report_bad_ip(ssl_sock.ip) # connect_control.fall_into_honeypot() raise socket.error(' certficate is none') if len(certs) < 3: # google_ip.report_connect_fail(ip, force_remove=True) raise Cert_Exception('No intermediate CA was found.') if hasattr(OpenSSL.crypto, "dump_publickey"): # old OpenSSL not support this function. if OpenSSL.crypto.dump_publickey(OpenSSL.crypto.FILETYPE_PEM, certs[1].get_pubkey()) not in GoogleG23PKP: # google_ip.report_connect_fail(ip, force_remove=True) raise Cert_Exception('The intermediate CA is mismatching.') issuer_commonname = next((v for k, v in certs[0].get_issuer().get_components() if k == 'CN'), '') if not issuer_commonname.startswith('Google'): # google_ip.report_connect_fail(ip, force_remove=True) raise Cert_Exception(' certficate is issued by %r, not Google' % (issuer_commonname)) if check_cert: verify_SSL_certificate_issuer(ssl_sock) connct_time = int((time_connected - time_begin) * 1000) handshake_time = int((time_handshaked - time_connected) * 1000) #xlog.debug("conn: %d handshake:%d", connct_time, handshake_time) # sometimes, we want to use raw tcp socket directly(select/epoll), so setattr it to ssl socket. ssl_sock._sock = sock ssl_sock.connct_time = connct_time ssl_sock.handshake_time = handshake_time ssl_sock.fd = sock.fileno() ssl_sock.create_time = time_begin ssl_sock.last_use_time = time_begin ssl_sock.received_size = 0 ssl_sock.load = 0 ssl_sock.sni = sni ssl_sock.host = "" return ssl_sock
def run(self): req_range_begin = None req_range_end = None for k, v in self.headers.items(): # logger.debug("range req head:%s => %s", k, v) if k.lower() == "range": req_range_begin, req_range_end = tuple( x for x in re.search(r'bytes=(\d*)-(\d*)', v).group(1, 2)) # break response_headers = dict((k.title(), v) for k, v in self.response.headers.items()) content_range = response_headers['Content-Range'] res_begin, res_end, res_length = tuple(int(x) for x in re.search( r'bytes (\d+)-(\d+)/(\d+)', content_range).group(1, 2, 3)) self.req_begin = res_end + 1 if req_range_begin and req_range_end: self.req_end = int(req_range_end) else: self.req_end = res_length - 1 self.wait_begin = res_begin if self.wait_begin == 0 and self.req_end == res_length - 1: response_headers['Content-Length'] = str(res_length) del response_headers['Content-Range'] state_code = 200 else: response_headers['Content-Range'] = 'bytes %s-%s/%s' % ( res_begin, self.req_end, res_length) response_headers['Content-Length'] = str( self.req_end - res_begin + 1) state_code = 206 response_headers["Persist"] = "" response_headers["Connection"] = "Persist" logger.info('RangeFetch %d-%d started(%r) ', res_begin, self.req_end, self.url) try: self.wfile.write("HTTP/1.1 %d OK\r\n" % state_code) for key in response_headers: if key in skip_response_headers: continue value = response_headers[key] #logger.debug("Head %s: %s", key.title(), value) send_header(self.wfile, key, value) self.wfile.write("\r\n") except Exception as e: self.keep_running = False logger.info("RangeFetch send response fail:%r %s", e, self.url) return data_left_to_fetch = self.req_end - self.req_begin + 1 fetch_times = int( (data_left_to_fetch + front.config.AUTORANGE_MAXSIZE - 1) / front.config.AUTORANGE_MAXSIZE) thread_num = min(front.config.AUTORANGE_THREADS, fetch_times) for i in xrange(0, thread_num): threading.Thread(target=self.fetch_worker).start() threading.Thread(target=self.fetch, args=( res_begin, res_end, self.response)).start() ok = "ok" while self.keep_running and \ (front.config.use_ipv6 == "force_ipv6" and \ check_local_network.IPv6.is_ok() or \ front.config.use_ipv6 != "force_ipv6" and \ check_local_network.is_ok()) and \ self.wait_begin < self.req_end + 1: with self.lock: if self.wait_begin not in self.data_list: self.waiter.wait() if self.wait_begin not in self.data_list: #logger.error("get notify but no data") continue else: data = self.data_list[self.wait_begin] del self.data_list[self.wait_begin] self.wait_begin += len(data) self.data_size -= len(data) self.all_data_size[self] = self.data_size try: ret = self.wfile.write(data) if ret == ssl.SSL_ERROR_WANT_WRITE or ret == ssl.SSL_ERROR_WANT_READ: logger.debug( "send to browser wfile.write ret:%d, retry", ret) ret = self.wfile.write(data) logger.debug("send to browser wfile.write ret:%d", ret) del data except Exception as e: logger.info('RangeFetch client closed(%s). %s', e, self.url) ok = None break self.keep_running = False self.all_data_size.pop(self, None) return ok
def connect_ssl(ip, port=443, timeout=5, check_cert=True, close_cb=None): if check_local_network.is_ok(ip): with network_fail_lock: time.sleep(0.1) ip_port = (ip, port) sni = sni_generater.get() if config.PROXY_ENABLE: sock = socks.socksocket(socket.AF_INET if ':' not in ip else socket.AF_INET6) else: sock = socket.socket(socket.AF_INET if ':' not in ip else socket.AF_INET6) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0)) # resize socket recv buffer 8K->32K to improve browser releated application performance sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32 * 1024) sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True) sock.settimeout(timeout) ssl_sock = openssl_wrap.SSLConnection(openssl_context, sock, ip, close_cb) ssl_sock.set_connect_state() ssl_sock.set_tlsext_host_name(sni) time_begin = time.time() ssl_sock.connect(ip_port) time_connected = time.time() ssl_sock.do_handshake() try: h2 = ssl_sock.get_alpn_proto_negotiated() if h2 == "h2": ssl_sock.h2 = True else: ssl_sock.h2 = False except Exception as e: if hasattr(ssl_sock._connection, "protos") and ssl_sock._connection.protos == "h2": ssl_sock.h2 = True else: ssl_sock.h2 = False time_handshaked = time.time() # report network ok check_local_network.report_ok(ip) cert = ssl_sock.get_peer_certificate() if not cert: raise socket.error(' certficate is none') if check_cert: issuer_commonname = next( (v for k, v in cert.get_issuer().get_components() if k == 'CN'), '') if not issuer_commonname.startswith('Google'): raise socket.error(' certficate is issued by %r, not Google' % (issuer_commonname)) connct_time = int((time_connected - time_begin) * 1000) handshake_time = int((time_handshaked - time_connected) * 1000) # sometimes, we want to use raw tcp socket directly(select/epoll), so setattr it to ssl socket. ssl_sock._sock = sock ssl_sock.connct_time = connct_time ssl_sock.handshake_time = handshake_time ssl_sock.fd = sock.fileno() ssl_sock.create_time = time_begin ssl_sock.last_use_time = time_begin ssl_sock.received_size = 0 ssl_sock.load = 0 ssl_sock.sni = sni ssl_sock.host = "" return ssl_sock
def connect_ssl(ip, port=443, timeout=5, check_cert=True, close_cb=None): if not check_local_network.is_ok(ip): with network_fail_lock: time.sleep(0.1) ip_port = (ip, port) sni = sni_generater.get() if int(config.PROXY_ENABLE): sock = socks.socksocket(socket.AF_INET if ':' not in ip else socket.AF_INET6) else: sock = socket.socket(socket.AF_INET if ':' not in ip else socket.AF_INET6) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0)) # resize socket recv buffer 8K->32K to improve browser releated application performance sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32 * 1024) sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True) sock.settimeout(timeout) ssl_sock = openssl_wrap.SSLConnection(openssl_context, sock, ip, close_cb) ssl_sock.set_connect_state() if hasattr(ssl_sock, 'set_tlsext_host_name'): try: ssl_sock.set_tlsext_host_name(sni) except: pass time_begin = time.time() ssl_sock.connect(ip_port) time_connected = time.time() ssl_sock.do_handshake() try: h2 = ssl_sock.get_alpn_proto_negotiated() if h2 == "h2": ssl_sock.h2 = True else: ssl_sock.h2 = False xlog.debug("%s alpn h2:%s", ip, h2) except Exception as e: #xlog.exception("alpn:%r", e) if hasattr(ssl_sock._connection, "protos") and ssl_sock._connection.protos == "h2": ssl_sock.h2 = True # xlog.debug("ip:%s http/2", ip) else: ssl_sock.h2 = False # xlog.debug("ip:%s http/1.1", ip) time_handshaked = time.time() check_local_network.report_ok(ip) def verify_SSL_certificate_issuer(ssl_sock): # cert = ssl_sock.get_peer_certificate() # if not cert: # #google_ip.report_bad_ip(ssl_sock.ip) # #connect_control.fall_into_honeypot() # raise socket.error(' certficate is none') # issuer_commonname = next((v for k, v in cert.get_issuer().get_components() if k == 'CN'), '') # if not issuer_commonname.startswith('Google'): # google_ip.report_connect_fail(ip, force_remove=True) # raise socket.error(' certficate is issued by %r, not Google' % ( issuer_commonname)) certs = ssl_sock.get_peer_cert_chain() if not certs: # google_ip.report_bad_ip(ssl_sock.ip) # connect_control.fall_into_honeypot() raise socket.error(' certficate is none') if len(certs) < 3: # google_ip.report_connect_fail(ip, force_remove=True) raise Cert_Exception('No intermediate CA was found.') if hasattr(OpenSSL.crypto, "dump_publickey"): # old OpenSSL not support this function. if OpenSSL.crypto.dump_publickey( OpenSSL.crypto.FILETYPE_PEM, certs[1].get_pubkey()) not in GoogleG23PKP: # google_ip.report_connect_fail(ip, force_remove=True) raise Cert_Exception('The intermediate CA is mismatching.') issuer_commonname = next( (v for k, v in certs[0].get_issuer().get_components() if k == 'CN'), '') if not issuer_commonname.startswith('Google'): # google_ip.report_connect_fail(ip, force_remove=True) raise Cert_Exception(' certficate is issued by %r, not Google' % (issuer_commonname)) if check_cert: verify_SSL_certificate_issuer(ssl_sock) connct_time = int((time_connected - time_begin) * 1000) handshake_time = int((time_handshaked - time_connected) * 1000) #xlog.debug("conn: %d handshake:%d", connct_time, handshake_time) # sometimes, we want to use raw tcp socket directly(select/epoll), so setattr it to ssl socket. ssl_sock._sock = sock ssl_sock.connct_time = connct_time ssl_sock.handshake_time = handshake_time ssl_sock.fd = sock.fileno() ssl_sock.create_time = time_begin ssl_sock.last_use_time = time_begin ssl_sock.received_size = 0 ssl_sock.load = 0 ssl_sock.sni = sni ssl_sock.host = "" return ssl_sock
def connect_ssl(ip, port=443, timeout=5, check_cert=True, close_cb=None): if check_local_network.is_ok(ip): with network_fail_lock: time.sleep(0.1) ip_port = (ip, port) sni = sni_generater.get() if config.PROXY_ENABLE: sock = socks.socksocket(socket.AF_INET if ':' not in ip else socket.AF_INET6) else: sock = socket.socket(socket.AF_INET if ':' not in ip else socket.AF_INET6) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0)) # resize socket recv buffer 8K->32K to improve browser releated application performance sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32*1024) sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True) sock.settimeout(timeout) ssl_sock = openssl_wrap.SSLConnection(openssl_context, sock, ip, close_cb) ssl_sock.set_connect_state() ssl_sock.set_tlsext_host_name(sni) time_begin = time.time() ssl_sock.connect(ip_port) time_connected = time.time() ssl_sock.do_handshake() try: h2 = ssl_sock.get_alpn_proto_negotiated() if h2 == "h2": ssl_sock.h2 = True else: ssl_sock.h2 = False except Exception as e: if hasattr(ssl_sock._connection, "protos") and ssl_sock._connection.protos == "h2": ssl_sock.h2 = True else: ssl_sock.h2 = False time_handshaked = time.time() # report network ok check_local_network.report_ok(ip) cert = ssl_sock.get_peer_certificate() if not cert: raise socket.error(' certficate is none') if check_cert: issuer_commonname = next((v for k, v in cert.get_issuer().get_components() if k == 'CN'), '') if not issuer_commonname.startswith('Google'): raise socket.error(' certficate is issued by %r, not Google' % ( issuer_commonname)) connct_time = int((time_connected - time_begin) * 1000) handshake_time = int((time_handshaked - time_connected) * 1000) # sometimes, we want to use raw tcp socket directly(select/epoll), so setattr it to ssl socket. ssl_sock._sock = sock ssl_sock.connct_time = connct_time ssl_sock.handshake_time = handshake_time ssl_sock.fd = sock.fileno() ssl_sock.create_time = time_begin ssl_sock.last_use_time = time_begin ssl_sock.received_size = 0 ssl_sock.load = 0 ssl_sock.sni = sni ssl_sock.host = "" return ssl_sock
def get_gws_ip(self): if not check_local_network.is_ok(): return 404 self.try_sort_gws_ip() self.ip_lock.acquire() try: ip_num = len(self.gws_ip_list) if ip_num == 0: #xlog.warning("no gws ip") #time.sleep(10) return None for i in range(ip_num): time_now = time.time() if self.gws_ip_pointer >= ip_num: if time_now - self.gws_ip_pointer_reset_time < 1: time.sleep(1) continue else: self.gws_ip_pointer = 0 self.gws_ip_pointer_reset_time = time_now elif self.gws_ip_pointer > 0 and time_now - self.gws_ip_pointer_reset_time > 3: self.gws_ip_pointer = 0 self.gws_ip_pointer_reset_time = time_now ip = self.gws_ip_list[self.gws_ip_pointer] get_time = self.ip_dict[ip]["get_time"] if time_now - get_time < self.ip_connect_interval: self.gws_ip_pointer += 1 continue if time_now - self.ip_dict[ip]['success_time'] > 300: # 5 min fail_connect_interval = 1800 # 30 min else: fail_connect_interval = 120 # 2 min fail_time = self.ip_dict[ip]["fail_time"] if time_now - fail_time < fail_connect_interval: self.gws_ip_pointer += 1 continue down_fail_connect_interval = 600 down_fail_time = self.ip_dict[ip]["down_fail_time"] if time_now - down_fail_time < down_fail_connect_interval: self.gws_ip_pointer += 1 continue if self.ip_dict[ip]['links'] >= config.max_links_per_ip: self.gws_ip_pointer += 1 continue handshake_time = self.ip_dict[ip]["handshake_time"] xlog.debug("get ip:%s t:%d", ip, handshake_time) self.ip_dict[ip]['get_time'] = time_now self.ip_dict[ip]['links'] += 1 self.gws_ip_pointer += 1 return ip except Exception as e: xlog.exception("get_gws_ip fail:%r", e) finally: self.ip_lock.release()