def report_connect_fail(self, ip, force_remove=False):
self.ip_lock.acquire()
try:
time_now = time.time()
if not ip in self.ip_dict:
xlog.debug("report_connect_fail %s not exist", ip)
return
if force_remove:
if self.ip_dict[ip]['fail_times'] == 0:
self.good_ip_num -= 1
self.bad_ip_num += 1
del self.ip_dict[ip]
if ip in self.gws_ip_list:
self.gws_ip_list.remove(ip)
xlog.info("remove ip:%s left amount:%d gws_num:%d", ip, len(self.ip_dict), len(self.gws_ip_list))
return
self.ip_dict[ip]['links'] -= 1
# ignore if system network is disconnected.
if not check_local_network.is_ok():
xlog.debug("report_connect_fail network fail")
return
check_local_network.report_network_fail()
if not check_local_network.is_ok():
return
fail_time = self.ip_dict[ip]["fail_time"]
if time_now - fail_time < 1:
xlog.debug("fail time too near %s", ip)
return
if self.ip_dict[ip]['fail_times'] == 0:
self.good_ip_num -= 1
self.bad_ip_num += 1
self.ip_dict[ip]['fail_times'] += 1
self.append_ip_history(ip, "fail")
self.ip_dict[ip]["fail_time"] = time_now
self.to_check_ip_queue.put((ip, time_now + 10))
xlog.debug("report_connect_fail:%s", ip)
except Exception as e:
xlog.exception("report_connect_fail err:%s", e)
finally:
self.iplist_need_save = True
self.ip_lock.release()
if not self.is_ip_enough():
self.search_more_google_ip()
def check_ip_process(self):
while connect_control.keep_running:
try:
ip, test_time = self.to_check_ip_queue.get()
except:
continue
time_wait = test_time - time.time()
if time_wait > 0:
time.sleep(time_wait)
if not check_local_network.is_ok():
try:
if self.ip_dict[ip]['fail_times']:
self.ip_dict[ip]['fail_times'] = 0
self.good_ip_num += 1
except:
pass
continue
result = check_ip.test_xtunnel_ip2(ip)
if result and result.support_xtunnel:
self.add_ip(ip, result.request_time, result.domain, "gws")
xlog.debug("restore ip:%s", ip)
continue
xlog.debug("ip:%s real fail", ip)
def check_ip_process(self):
while connect_control.keep_running:
try:
ip, test_time = self.to_check_ip_queue.get()
except:
continue
time_wait = test_time - time.time()
if time_wait > 0:
time.sleep(time_wait)
if not check_local_network.is_ok():
try:
if self.ip_dict[ip]['fail_times']:
self.ip_dict[ip]['fail_times'] = 0
self.good_ip_num += 1
self.bad_ip_num -= 1
except:
pass
continue
result = check_ip.test_gae_ip2(ip)
if result and result.support_gae:
self.add_ip(ip, result.handshake_time, result.domain, "gws")
xlog.debug("restore ip:%s", ip)
continue
xlog.debug("ip:%s real fail", ip)
def do_METHOD(self):
self.req_payload = None
host = self.headers.get('Host', '')
host_ip, _, port = host.rpartition(':')
if self.is_local([host, host_ip]):
logger.debug("Browse localhost by proxy")
return self.forward_local()
elif host == self.fake_host:
# logger.debug("%s %s", self.command, self.path)
# for web_ui status page
# auto detect browser proxy setting is work
return self.wfile.write(self.self_check_response_data)
if not (front.config.use_ipv6 == "force_ipv6" and \
check_local_network.IPv6.is_ok() or \
front.config.use_ipv6 != "force_ipv6" and \
check_local_network.is_ok()):
self.close_connection = 1
return
if isinstance(self.connection, ssl.SSLSocket):
schema = "https"
else:
schema = "http"
if self.path[0] == '/':
self.host = self.headers['Host']
self.url = '%s://%s%s' % (schema, host, self.path)
else:
self.url = self.path
self.parsed_url = urlparse.urlparse(self.path)
self.host = self.parsed_url[1]
if len(self.parsed_url[4]):
self.path = '?'.join([self.parsed_url[2], self.parsed_url[4]])
else:
self.path = self.parsed_url[2]
if len(self.url) > 2083 and self.host.endswith(
front.config.GOOGLE_ENDSWITH):
return self.go_DIRECT()
if self.host in front.config.HOSTS_GAE:
return self.go_AGENT()
# redirect http request to https request
# avoid key word filter when pass through GFW
if host in front.config.HOSTS_DIRECT:
return self.go_DIRECT()
if host.endswith(front.config.HOSTS_GAE_ENDSWITH):
return self.go_AGENT()
if host.endswith(front.config.HOSTS_DIRECT_ENDSWITH):
return self.go_DIRECT()
return self.go_AGENT()
def do_METHOD(self):
self.req_payload = None
host = self.headers.get('Host', '')
host_ip, _, port = host.rpartition(':')
if self.is_local([host, host_ip]):
xlog.debug("Browse localhost by proxy")
return self.forward_local()
elif host == self.fake_host:
# xlog.debug("%s %s", self.command, self.path)
# for web_ui status page
# auto detect browser proxy setting is work
return self.wfile.write(self.self_check_response_data)
if not (front.config.use_ipv6 == "force_ipv6" and \
check_local_network.IPv6.is_ok() or \
front.config.use_ipv6 != "force_ipv6" and \
check_local_network.is_ok()):
self.close_connection = 1
return
if isinstance(self.connection, ssl.SSLSocket):
schema = "https"
else:
schema = "http"
if self.path[0] == '/':
self.host = self.headers['Host']
self.url = '%s://%s%s' % (schema, host, self.path)
else:
self.url = self.path
self.parsed_url = urlparse.urlparse(self.path)
self.host = self.parsed_url[1]
if len(self.parsed_url[4]):
self.path = '?'.join([self.parsed_url[2], self.parsed_url[4]])
else:
self.path = self.parsed_url[2]
if len(self.url) > 2083 and self.host.endswith(front.config.GOOGLE_ENDSWITH):
return self.go_DIRECT()
if self.host in front.config.HOSTS_GAE:
return self.go_AGENT()
# redirect http request to https request
# avoid key word filter when pass through GFW
if host in front.config.HOSTS_DIRECT:
return self.go_DIRECT()
if host.endswith(front.config.HOSTS_GAE_ENDSWITH):
return self.go_AGENT()
if host.endswith(front.config.HOSTS_DIRECT_ENDSWITH):
return self.go_DIRECT()
return self.go_AGENT()
def _create_ssl_connection(self, ip_port):
if not connect_control.allow_connect():
time.sleep(10)
return False
ip = ip_port[0]
port = ip_port[1]
connect_control.start_connect_register(high_prior=True)
time_begin = time.time()
try:
xlog.debug("try connect %s", ip)
ssl_sock = check_ip.connect_ssl(ip,
port=port,
timeout=self.connect_timeout,
on_close=ip_manager.ssl_closed)
ip_manager.update_ip(ip, ssl_sock.handshake_time)
xlog.debug("create_ssl update ip:%s time:%d h2:%d sni:%s top:%s",
ip, ssl_sock.handshake_time, ssl_sock.h2, ssl_sock.sni,
ssl_sock.top_domain)
ssl_sock.last_use_time = ssl_sock.create_time
ssl_sock.received_size = 0
ssl_sock.load = 0
ssl_sock.host = self.sub + "." + ssl_sock.top_domain
connect_control.report_connect_success()
return ssl_sock
except Exception as e:
time_cost = time.time() - time_begin
if time_cost < self.connect_timeout - 1:
xlog.debug("connect %s fail:%s cost:%d ", ip, e,
time_cost * 1000)
else:
xlog.debug("%s fail:%r", ip, e)
ip_manager.report_connect_fail(ip)
connect_control.report_connect_fail()
if not check_local_network.is_ok():
time.sleep(10)
else:
time.sleep(1)
return False
finally:
connect_control.end_connect_register(high_prior=True)
def recheck_ip(self, ip):
# recheck ip if not work.
# can block.
if not check_local_network.is_ok():
xlog.debug("recheck_ip:%s network is fail", ip)
return
self.report_connect_fail(ip)
connect_control.start_connect_register()
result = check_ip.test_gae_ip2(ip)
connect_control.end_connect_register()
if not result:
self.report_connect_fail(ip, force_remove=True)
xlog.debug("recheck_ip:%s real fail, removed.", ip)
else:
self.add_ip(ip, result.handshake_time, result.domain, "gws")
xlog.debug("recheck_ip:%s restore okl", ip)
def recheck_ip(self, ip):
# recheck ip if not work.
# can block.
if not check_local_network.is_ok():
xlog.debug("recheck_ip:%s network is fail", ip)
return
self.report_connect_fail(ip)
result = check_ip.test_gae_ip2(ip)
if not result:
# connect fail.
# do nothing
return
if not result.support_gae:
self.report_connect_fail(ip, force_remove=True)
xlog.debug("recheck_ip:%s real fail, removed.", ip)
else:
self.add_ip(ip, result.handshake_time, result.domain, "gws")
xlog.debug("recheck_ip:%s restore okl", ip)
def _create_ssl_connection(self, ip_port):
if not connect_control.allow_connect():
time.sleep(10)
return False
ip = ip_port[0]
port = ip_port[1]
connect_control.start_connect_register(high_prior=True)
time_begin = time.time()
try:
xlog.debug("try connect %s", ip)
ssl_sock = check_ip.connect_ssl(ip, port=port, timeout=self.connect_timeout, on_close=ip_manager.ssl_closed)
ip_manager.update_ip(ip, ssl_sock.handshake_time)
xlog.debug("create_ssl update ip:%s time:%d h2:%d sni:%s top:%s",
ip, ssl_sock.handshake_time, ssl_sock.h2, ssl_sock.sni, ssl_sock.top_domain)
ssl_sock.last_use_time = ssl_sock.create_time
ssl_sock.received_size = 0
ssl_sock.load = 0
ssl_sock.host = self.sub + "." + ssl_sock.top_domain
connect_control.report_connect_success()
return ssl_sock
except Exception as e:
time_cost = time.time() - time_begin
if time_cost < self.connect_timeout - 1:
xlog.debug("connect %s fail:%s cost:%d ", ip, e, time_cost * 1000)
else:
xlog.debug("%s fail:%r", ip, e)
ip_manager.report_connect_fail(ip)
connect_control.report_connect_fail()
if not check_local_network.is_ok():
time.sleep(10)
else:
time.sleep(1)
return False
finally:
connect_control.end_connect_register(high_prior=True)
def recheck_ip(self, ip):
# recheck ip if not work.
# can block.
if not check_local_network.is_ok():
xlog.debug("recheck_ip:%s network is fail", ip)
return
self.report_connect_fail(ip)
connect_control.start_connect_register()
result = check_ip.test_xtunnel_ip2(ip)
connect_control.end_connect_register()
if not result:
# connect fail.
# do nothing
return
if not result.support_xtunnel:
self.report_connect_fail(ip, force_remove=True)
xlog.debug("recheck_ip:%s real fail, removed.", ip)
else:
self.add_ip(ip, result.request_time, result.domain, "gws")
xlog.debug("recheck_ip:%s restore okl", ip)
def run(self):
req_range_begin = None
req_range_end = None
for k, v in self.headers.items():
# xlog.debug("range req head:%s => %s", k, v)
if k.lower() == "range":
req_range_begin, req_range_end = tuple(
x for x in re.search(r'bytes=(\d*)-(\d*)', v).group(1, 2))
# break
response_headers = dict((k.title(), v)
for k, v in self.response.headers.items())
content_range = response_headers['Content-Range']
res_begin, res_end, res_length = tuple(int(x) for x in re.search(
r'bytes (\d+)-(\d+)/(\d+)', content_range).group(1, 2, 3))
self.req_begin = res_end + 1
if req_range_begin and req_range_end:
self.req_end = int(req_range_end)
else:
self.req_end = res_length - 1
self.wait_begin = res_begin
if self.wait_begin == 0 and self.req_end == res_length - 1:
response_headers['Content-Length'] = str(res_length)
del response_headers['Content-Range']
state_code = 200
else:
response_headers['Content-Range'] = 'bytes %s-%s/%s' % (
res_begin, self.req_end, res_length)
response_headers['Content-Length'] = str(
self.req_end - res_begin + 1)
state_code = 206
response_headers["Persist"] = ""
response_headers["Connection"] = "Persist"
xlog.info('RangeFetch %d-%d started(%r) ',
res_begin, self.req_end, self.url)
try:
self.wfile.write("HTTP/1.1 %d OK\r\n" % state_code)
for key in response_headers:
if key in skip_response_headers:
continue
value = response_headers[key]
#xlog.debug("Head %s: %s", key.title(), value)
send_header(self.wfile, key, value)
self.wfile.write("\r\n")
except Exception as e:
self.keep_running = False
xlog.info("RangeFetch send response fail:%r %s", e, self.url)
return
data_left_to_fetch = self.req_end - self.req_begin + 1
fetch_times = int(
(data_left_to_fetch + front.config.AUTORANGE_MAXSIZE - 1) / front.config.AUTORANGE_MAXSIZE)
thread_num = min(front.config.AUTORANGE_THREADS, fetch_times)
for i in xrange(0, thread_num):
threading.Thread(target=self.fetch_worker).start()
threading.Thread(target=self.fetch, args=(
res_begin, res_end, self.response)).start()
ok = "ok"
while self.keep_running and \
(front.config.use_ipv6 == "force_ipv6" and \
check_local_network.IPv6.is_ok() or \
front.config.use_ipv6 != "force_ipv6" and \
check_local_network.is_ok()) and \
self.wait_begin < self.req_end + 1:
with self.lock:
if self.wait_begin not in self.data_list:
self.waiter.wait()
if self.wait_begin not in self.data_list:
xlog.error("get notify but no data")
continue
else:
data = self.data_list[self.wait_begin]
del self.data_list[self.wait_begin]
self.wait_begin += len(data)
self.data_size -= len(data)
self.all_data_size[self] = self.data_size
try:
ret = self.wfile.write(data)
if ret == ssl.SSL_ERROR_WANT_WRITE or ret == ssl.SSL_ERROR_WANT_READ:
xlog.debug(
"send to browser wfile.write ret:%d, retry", ret)
ret = self.wfile.write(data)
xlog.debug("send to browser wfile.write ret:%d", ret)
del data
except Exception as e:
xlog.info('RangeFetch client closed(%s). %s', e, self.url)
ok = None
break
self.keep_running = False
self.all_data_size.pop(self, None)
return ok
def connect_ssl(ip, port=443, timeout=5, check_cert=True, close_cb=None):
if not check_local_network.is_ok(ip):
with network_fail_lock:
time.sleep(0.1)
ip_port = (ip, port)
sni = sni_generater.get()
if int(config.PROXY_ENABLE):
sock = socks.socksocket(socket.AF_INET if ':' not in ip else socket.AF_INET6)
else:
sock = socket.socket(socket.AF_INET if ':' not in ip else socket.AF_INET6)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0))
# resize socket recv buffer 8K->32K to improve browser releated application performance
sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32*1024)
sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
sock.settimeout(timeout)
ssl_sock = openssl_wrap.SSLConnection(openssl_context, sock, ip, close_cb)
ssl_sock.set_connect_state()
if hasattr(ssl_sock, 'set_tlsext_host_name'):
try:
ssl_sock.set_tlsext_host_name(sni)
except:
pass
time_begin = time.time()
ssl_sock.connect(ip_port)
time_connected = time.time()
ssl_sock.do_handshake()
try:
h2 = ssl_sock.get_alpn_proto_negotiated()
if h2 == "h2":
ssl_sock.h2 = True
else:
ssl_sock.h2 = False
xlog.debug("%s alpn h2:%s", ip, h2)
except Exception as e:
#xlog.exception("alpn:%r", e)
if hasattr(ssl_sock._connection, "protos") and ssl_sock._connection.protos == "h2":
ssl_sock.h2 = True
# xlog.debug("ip:%s http/2", ip)
else:
ssl_sock.h2 = False
# xlog.debug("ip:%s http/1.1", ip)
time_handshaked = time.time()
check_local_network.report_ok(ip)
def verify_SSL_certificate_issuer(ssl_sock):
# cert = ssl_sock.get_peer_certificate()
# if not cert:
# #google_ip.report_bad_ip(ssl_sock.ip)
# #connect_control.fall_into_honeypot()
# raise socket.error(' certficate is none')
# issuer_commonname = next((v for k, v in cert.get_issuer().get_components() if k == 'CN'), '')
# if not issuer_commonname.startswith('Google'):
# google_ip.report_connect_fail(ip, force_remove=True)
# raise socket.error(' certficate is issued by %r, not Google' % ( issuer_commonname))
certs = ssl_sock.get_peer_cert_chain()
if not certs:
# google_ip.report_bad_ip(ssl_sock.ip)
# connect_control.fall_into_honeypot()
raise socket.error(' certficate is none')
if len(certs) < 3:
# google_ip.report_connect_fail(ip, force_remove=True)
raise Cert_Exception('No intermediate CA was found.')
if hasattr(OpenSSL.crypto, "dump_publickey"):
# old OpenSSL not support this function.
if OpenSSL.crypto.dump_publickey(OpenSSL.crypto.FILETYPE_PEM, certs[1].get_pubkey()) not in GoogleG23PKP:
# google_ip.report_connect_fail(ip, force_remove=True)
raise Cert_Exception('The intermediate CA is mismatching.')
issuer_commonname = next((v for k, v in certs[0].get_issuer().get_components() if k == 'CN'), '')
if not issuer_commonname.startswith('Google'):
# google_ip.report_connect_fail(ip, force_remove=True)
raise Cert_Exception(' certficate is issued by %r, not Google' % (issuer_commonname))
if check_cert:
verify_SSL_certificate_issuer(ssl_sock)
connct_time = int((time_connected - time_begin) * 1000)
handshake_time = int((time_handshaked - time_connected) * 1000)
#xlog.debug("conn: %d handshake:%d", connct_time, handshake_time)
# sometimes, we want to use raw tcp socket directly(select/epoll), so setattr it to ssl socket.
ssl_sock._sock = sock
ssl_sock.connct_time = connct_time
ssl_sock.handshake_time = handshake_time
ssl_sock.fd = sock.fileno()
ssl_sock.create_time = time_begin
ssl_sock.last_use_time = time_begin
ssl_sock.received_size = 0
ssl_sock.load = 0
ssl_sock.sni = sni
ssl_sock.host = ""
return ssl_sock
def run(self):
req_range_begin = None
req_range_end = None
for k, v in self.headers.items():
# logger.debug("range req head:%s => %s", k, v)
if k.lower() == "range":
req_range_begin, req_range_end = tuple(
x for x in re.search(r'bytes=(\d*)-(\d*)', v).group(1, 2))
# break
response_headers = dict((k.title(), v)
for k, v in self.response.headers.items())
content_range = response_headers['Content-Range']
res_begin, res_end, res_length = tuple(int(x) for x in re.search(
r'bytes (\d+)-(\d+)/(\d+)', content_range).group(1, 2, 3))
self.req_begin = res_end + 1
if req_range_begin and req_range_end:
self.req_end = int(req_range_end)
else:
self.req_end = res_length - 1
self.wait_begin = res_begin
if self.wait_begin == 0 and self.req_end == res_length - 1:
response_headers['Content-Length'] = str(res_length)
del response_headers['Content-Range']
state_code = 200
else:
response_headers['Content-Range'] = 'bytes %s-%s/%s' % (
res_begin, self.req_end, res_length)
response_headers['Content-Length'] = str(
self.req_end - res_begin + 1)
state_code = 206
response_headers["Persist"] = ""
response_headers["Connection"] = "Persist"
logger.info('RangeFetch %d-%d started(%r) ',
res_begin, self.req_end, self.url)
try:
self.wfile.write("HTTP/1.1 %d OK\r\n" % state_code)
for key in response_headers:
if key in skip_response_headers:
continue
value = response_headers[key]
#logger.debug("Head %s: %s", key.title(), value)
send_header(self.wfile, key, value)
self.wfile.write("\r\n")
except Exception as e:
self.keep_running = False
logger.info("RangeFetch send response fail:%r %s", e, self.url)
return
data_left_to_fetch = self.req_end - self.req_begin + 1
fetch_times = int(
(data_left_to_fetch + front.config.AUTORANGE_MAXSIZE - 1) / front.config.AUTORANGE_MAXSIZE)
thread_num = min(front.config.AUTORANGE_THREADS, fetch_times)
for i in xrange(0, thread_num):
threading.Thread(target=self.fetch_worker).start()
threading.Thread(target=self.fetch, args=(
res_begin, res_end, self.response)).start()
ok = "ok"
while self.keep_running and \
(front.config.use_ipv6 == "force_ipv6" and \
check_local_network.IPv6.is_ok() or \
front.config.use_ipv6 != "force_ipv6" and \
check_local_network.is_ok()) and \
self.wait_begin < self.req_end + 1:
with self.lock:
if self.wait_begin not in self.data_list:
self.waiter.wait()
if self.wait_begin not in self.data_list:
#logger.error("get notify but no data")
continue
else:
data = self.data_list[self.wait_begin]
del self.data_list[self.wait_begin]
self.wait_begin += len(data)
self.data_size -= len(data)
self.all_data_size[self] = self.data_size
try:
ret = self.wfile.write(data)
if ret == ssl.SSL_ERROR_WANT_WRITE or ret == ssl.SSL_ERROR_WANT_READ:
logger.debug(
"send to browser wfile.write ret:%d, retry", ret)
ret = self.wfile.write(data)
logger.debug("send to browser wfile.write ret:%d", ret)
del data
except Exception as e:
logger.info('RangeFetch client closed(%s). %s', e, self.url)
ok = None
break
self.keep_running = False
self.all_data_size.pop(self, None)
return ok
def connect_ssl(ip, port=443, timeout=5, check_cert=True, close_cb=None):
if check_local_network.is_ok(ip):
with network_fail_lock:
time.sleep(0.1)
ip_port = (ip, port)
sni = sni_generater.get()
if config.PROXY_ENABLE:
sock = socks.socksocket(socket.AF_INET if ':' not in
ip else socket.AF_INET6)
else:
sock = socket.socket(socket.AF_INET if ':' not in
ip else socket.AF_INET6)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER,
struct.pack('ii', 1, 0))
# resize socket recv buffer 8K->32K to improve browser releated application performance
sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32 * 1024)
sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
sock.settimeout(timeout)
ssl_sock = openssl_wrap.SSLConnection(openssl_context, sock, ip, close_cb)
ssl_sock.set_connect_state()
ssl_sock.set_tlsext_host_name(sni)
time_begin = time.time()
ssl_sock.connect(ip_port)
time_connected = time.time()
ssl_sock.do_handshake()
try:
h2 = ssl_sock.get_alpn_proto_negotiated()
if h2 == "h2":
ssl_sock.h2 = True
else:
ssl_sock.h2 = False
except Exception as e:
if hasattr(ssl_sock._connection,
"protos") and ssl_sock._connection.protos == "h2":
ssl_sock.h2 = True
else:
ssl_sock.h2 = False
time_handshaked = time.time()
# report network ok
check_local_network.report_ok(ip)
cert = ssl_sock.get_peer_certificate()
if not cert:
raise socket.error(' certficate is none')
if check_cert:
issuer_commonname = next(
(v for k, v in cert.get_issuer().get_components() if k == 'CN'),
'')
if not issuer_commonname.startswith('Google'):
raise socket.error(' certficate is issued by %r, not Google' %
(issuer_commonname))
connct_time = int((time_connected - time_begin) * 1000)
handshake_time = int((time_handshaked - time_connected) * 1000)
# sometimes, we want to use raw tcp socket directly(select/epoll), so setattr it to ssl socket.
ssl_sock._sock = sock
ssl_sock.connct_time = connct_time
ssl_sock.handshake_time = handshake_time
ssl_sock.fd = sock.fileno()
ssl_sock.create_time = time_begin
ssl_sock.last_use_time = time_begin
ssl_sock.received_size = 0
ssl_sock.load = 0
ssl_sock.sni = sni
ssl_sock.host = ""
return ssl_sock
def connect_ssl(ip, port=443, timeout=5, check_cert=True, close_cb=None):
if not check_local_network.is_ok(ip):
with network_fail_lock:
time.sleep(0.1)
ip_port = (ip, port)
sni = sni_generater.get()
if int(config.PROXY_ENABLE):
sock = socks.socksocket(socket.AF_INET if ':' not in
ip else socket.AF_INET6)
else:
sock = socket.socket(socket.AF_INET if ':' not in
ip else socket.AF_INET6)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER,
struct.pack('ii', 1, 0))
# resize socket recv buffer 8K->32K to improve browser releated application performance
sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32 * 1024)
sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
sock.settimeout(timeout)
ssl_sock = openssl_wrap.SSLConnection(openssl_context, sock, ip, close_cb)
ssl_sock.set_connect_state()
if hasattr(ssl_sock, 'set_tlsext_host_name'):
try:
ssl_sock.set_tlsext_host_name(sni)
except:
pass
time_begin = time.time()
ssl_sock.connect(ip_port)
time_connected = time.time()
ssl_sock.do_handshake()
try:
h2 = ssl_sock.get_alpn_proto_negotiated()
if h2 == "h2":
ssl_sock.h2 = True
else:
ssl_sock.h2 = False
xlog.debug("%s alpn h2:%s", ip, h2)
except Exception as e:
#xlog.exception("alpn:%r", e)
if hasattr(ssl_sock._connection,
"protos") and ssl_sock._connection.protos == "h2":
ssl_sock.h2 = True
# xlog.debug("ip:%s http/2", ip)
else:
ssl_sock.h2 = False
# xlog.debug("ip:%s http/1.1", ip)
time_handshaked = time.time()
check_local_network.report_ok(ip)
def verify_SSL_certificate_issuer(ssl_sock):
# cert = ssl_sock.get_peer_certificate()
# if not cert:
# #google_ip.report_bad_ip(ssl_sock.ip)
# #connect_control.fall_into_honeypot()
# raise socket.error(' certficate is none')
# issuer_commonname = next((v for k, v in cert.get_issuer().get_components() if k == 'CN'), '')
# if not issuer_commonname.startswith('Google'):
# google_ip.report_connect_fail(ip, force_remove=True)
# raise socket.error(' certficate is issued by %r, not Google' % ( issuer_commonname))
certs = ssl_sock.get_peer_cert_chain()
if not certs:
# google_ip.report_bad_ip(ssl_sock.ip)
# connect_control.fall_into_honeypot()
raise socket.error(' certficate is none')
if len(certs) < 3:
# google_ip.report_connect_fail(ip, force_remove=True)
raise Cert_Exception('No intermediate CA was found.')
if hasattr(OpenSSL.crypto, "dump_publickey"):
# old OpenSSL not support this function.
if OpenSSL.crypto.dump_publickey(
OpenSSL.crypto.FILETYPE_PEM,
certs[1].get_pubkey()) not in GoogleG23PKP:
# google_ip.report_connect_fail(ip, force_remove=True)
raise Cert_Exception('The intermediate CA is mismatching.')
issuer_commonname = next(
(v
for k, v in certs[0].get_issuer().get_components() if k == 'CN'),
'')
if not issuer_commonname.startswith('Google'):
# google_ip.report_connect_fail(ip, force_remove=True)
raise Cert_Exception(' certficate is issued by %r, not Google' %
(issuer_commonname))
if check_cert:
verify_SSL_certificate_issuer(ssl_sock)
connct_time = int((time_connected - time_begin) * 1000)
handshake_time = int((time_handshaked - time_connected) * 1000)
#xlog.debug("conn: %d handshake:%d", connct_time, handshake_time)
# sometimes, we want to use raw tcp socket directly(select/epoll), so setattr it to ssl socket.
ssl_sock._sock = sock
ssl_sock.connct_time = connct_time
ssl_sock.handshake_time = handshake_time
ssl_sock.fd = sock.fileno()
ssl_sock.create_time = time_begin
ssl_sock.last_use_time = time_begin
ssl_sock.received_size = 0
ssl_sock.load = 0
ssl_sock.sni = sni
ssl_sock.host = ""
return ssl_sock
def connect_ssl(ip, port=443, timeout=5, check_cert=True, close_cb=None):
if check_local_network.is_ok(ip):
with network_fail_lock:
time.sleep(0.1)
ip_port = (ip, port)
sni = sni_generater.get()
if config.PROXY_ENABLE:
sock = socks.socksocket(socket.AF_INET if ':' not in ip else socket.AF_INET6)
else:
sock = socket.socket(socket.AF_INET if ':' not in ip else socket.AF_INET6)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0))
# resize socket recv buffer 8K->32K to improve browser releated application performance
sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32*1024)
sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
sock.settimeout(timeout)
ssl_sock = openssl_wrap.SSLConnection(openssl_context, sock, ip, close_cb)
ssl_sock.set_connect_state()
ssl_sock.set_tlsext_host_name(sni)
time_begin = time.time()
ssl_sock.connect(ip_port)
time_connected = time.time()
ssl_sock.do_handshake()
try:
h2 = ssl_sock.get_alpn_proto_negotiated()
if h2 == "h2":
ssl_sock.h2 = True
else:
ssl_sock.h2 = False
except Exception as e:
if hasattr(ssl_sock._connection, "protos") and ssl_sock._connection.protos == "h2":
ssl_sock.h2 = True
else:
ssl_sock.h2 = False
time_handshaked = time.time()
# report network ok
check_local_network.report_ok(ip)
cert = ssl_sock.get_peer_certificate()
if not cert:
raise socket.error(' certficate is none')
if check_cert:
issuer_commonname = next((v for k, v in cert.get_issuer().get_components() if k == 'CN'), '')
if not issuer_commonname.startswith('Google'):
raise socket.error(' certficate is issued by %r, not Google' % ( issuer_commonname))
connct_time = int((time_connected - time_begin) * 1000)
handshake_time = int((time_handshaked - time_connected) * 1000)
# sometimes, we want to use raw tcp socket directly(select/epoll), so setattr it to ssl socket.
ssl_sock._sock = sock
ssl_sock.connct_time = connct_time
ssl_sock.handshake_time = handshake_time
ssl_sock.fd = sock.fileno()
ssl_sock.create_time = time_begin
ssl_sock.last_use_time = time_begin
ssl_sock.received_size = 0
ssl_sock.load = 0
ssl_sock.sni = sni
ssl_sock.host = ""
return ssl_sock
def get_gws_ip(self):
if not check_local_network.is_ok():
return 404
self.try_sort_gws_ip()
self.ip_lock.acquire()
try:
ip_num = len(self.gws_ip_list)
if ip_num == 0:
#xlog.warning("no gws ip")
#time.sleep(10)
return None
for i in range(ip_num):
time_now = time.time()
if self.gws_ip_pointer >= ip_num:
if time_now - self.gws_ip_pointer_reset_time < 1:
time.sleep(1)
continue
else:
self.gws_ip_pointer = 0
self.gws_ip_pointer_reset_time = time_now
elif self.gws_ip_pointer > 0 and time_now - self.gws_ip_pointer_reset_time > 3:
self.gws_ip_pointer = 0
self.gws_ip_pointer_reset_time = time_now
ip = self.gws_ip_list[self.gws_ip_pointer]
get_time = self.ip_dict[ip]["get_time"]
if time_now - get_time < self.ip_connect_interval:
self.gws_ip_pointer += 1
continue
if time_now - self.ip_dict[ip]['success_time'] > 300: # 5 min
fail_connect_interval = 1800 # 30 min
else:
fail_connect_interval = 120 # 2 min
fail_time = self.ip_dict[ip]["fail_time"]
if time_now - fail_time < fail_connect_interval:
self.gws_ip_pointer += 1
continue
down_fail_connect_interval = 600
down_fail_time = self.ip_dict[ip]["down_fail_time"]
if time_now - down_fail_time < down_fail_connect_interval:
self.gws_ip_pointer += 1
continue
if self.ip_dict[ip]['links'] >= config.max_links_per_ip:
self.gws_ip_pointer += 1
continue
handshake_time = self.ip_dict[ip]["handshake_time"]
xlog.debug("get ip:%s t:%d", ip, handshake_time)
self.ip_dict[ip]['get_time'] = time_now
self.ip_dict[ip]['links'] += 1
self.gws_ip_pointer += 1
return ip
except Exception as e:
xlog.exception("get_gws_ip fail:%r", e)
finally:
self.ip_lock.release()
