def _create_ssl_connection(self, ip_port):
if not connect_control.allow_connect():
time.sleep(10)
return False
ip = ip_port[0]
port = ip_port[1]
connect_control.start_connect_register(high_prior=True)
time_begin = time.time()
try:
ssl_sock = check_ip.connect_ssl(ip, port=port, timeout=self.connect_timeout, on_close=ip_manager.ssl_closed)
xlog.debug("create_ssl update ip:%s time:%d h2:%d sni:%s top:%s",
ip, ssl_sock.handshake_time, ssl_sock.h2, ssl_sock.sni, ssl_sock.top_domain)
ssl_sock.last_use_time = ssl_sock.create_time
ssl_sock.received_size = 0
ssl_sock.load = 0
ssl_sock.host = self.sub + "." + ssl_sock.top_domain
connect_control.report_connect_success()
return ssl_sock
except Exception as e:
time_cost = time.time() - time_begin
if time_cost < self.connect_timeout - 1:
xlog.debug("connect %s fail:%s cost:%d ", ip, e, time_cost * 1000)
else:
xlog.debug("%s fail:%r", ip, e)
ip_manager.report_connect_fail(ip)
connect_control.report_connect_fail()
return False
finally:
connect_control.end_connect_register(high_prior=True)
def connect_thread(self, sleep_time=0):
time.sleep(sleep_time)
try:
while self.new_conn_pool.qsize() < self.https_new_connect_num or \
self.new_conn_pool.qsize(only_h1=True) < 1:
if self.new_conn_pool.qsize() > self.connection_pool_max_num:
break
if not connect_control.allow_connect():
break
ip_str = google_ip.get_gws_ip()
if not ip_str:
xlog.warning("no enough ip")
time.sleep(60)
break
ssl_sock = self._create_ssl_connection( (ip_str, 443) )
if not ssl_sock:
continue
if self.ssl_timeout_cb and \
self.new_conn_pool.qsize() > self.connection_pool_max_num + 1 and \
self.new_conn_pool.qsize() > self.https_new_connect_num + 1 and \
self.new_conn_pool.qsize(only_h1=True) > 2:
self.ssl_timeout_cb(ssl_sock)
else:
self.new_conn_pool.put((ssl_sock.handshake_time, ssl_sock))
time.sleep(1)
finally:
self.thread_num_lock.acquire()
self.thread_num -= 1
self.thread_num_lock.release()
def runJob(self):
while True: #not self.is_ip_enough() and self.searching_thread_count < 2:
if not connect_control.allow_connect():
break
try:
time.sleep(1)
ip_int = ip_range.get_ip()
ip_str = ip_utils.ip_num_to_string(ip_int)
if self.is_bad_ip(ip_str):
continue
result = check_ip.test_gws(ip_str)
if not result:
continue
if self.add_ip(ip_str, result.handshake_time, result.domain,
result.server_type):
#logging.info("add %s CN:%s type:%s time:%d gws:%d ", ip_str,
# result.domain, result.server_type, result.handshake_time, len(self.gws_ip_list))
logging.info("check_ip add ip:%s time:%d", ip_str,
result.handshake_time)
self.remove_slowest_ip()
self.save_ip_list()
except check_ip.HoneypotError as e:
self.report_bad_ip(ip_str)
connect_control.fall_into_honeypot()
break
except Exception as e:
logging.exception("google_ip.runJob fail:%s", e)
self.ncount_lock.acquire()
self.searching_thread_count -= 1
self.ncount_lock.release()
def connect_thread(self, sleep_time=0):
time.sleep(sleep_time)
try:
while self.new_conn_pool.qsize() < self.https_new_connect_num:
ip_str = google_ip.get_gws_ip()
if not ip_str:
time.sleep(60)
xlog.warning("no enough ip")
break
port = 443
#xlog.debug("create ssl conn %s", ip_str)
ssl_sock = self._create_ssl_connection( (ip_str, port) )
if ssl_sock:
ssl_sock.last_use_time = time.time()
if self.new_conn_pool.qsize() >= self.connection_pool_max_num and self.ssl_timeout_cb:
self.ssl_timeout_cb(ssl_sock)
else:
self.new_conn_pool.put((ssl_sock.handshake_time, ssl_sock))
elif not connect_control.allow_connect():
break
time.sleep(1)
finally:
self.thread_num_lock.acquire()
self.thread_num -= 1
self.thread_num_lock.release()
def create_connection_worker(self):
try:
while connect_control.keep_running:
if self.new_conn_pool.qsize() >= self.connection_pool_min_num:
#xlog.debug("get enough conn")
break
if self.thread_num > self.connection_pool_min_num - self.new_conn_pool.qsize():
break
ip_str = google_ip.get_gws_ip()
if not ip_str:
xlog.warning("no gws ip")
break
port = 443
#logging.debug("create ssl conn %s", ip_str)
connect_control.start_connect_register(True)
ssl_sock = self._create_ssl_connection( (ip_str, port) )
connect_control.end_connect_register(True)
if ssl_sock:
ssl_sock.last_use_time = time.time()
self.new_conn_pool.put((ssl_sock.handshake_time, ssl_sock))
elif not connect_control.allow_connect():
xlog.debug("create_connection_worker, control not allow")
time.sleep(10)
time.sleep(1)
finally:
self.thread_num_lock.acquire()
self.thread_num -= 1
self.thread_num_lock.release()
def create_connection_daemon(self):
connect_start_num = 0
while connect_control.keep_running:
time.sleep(0.1)
if not connect_control.allow_connect():
time.sleep(5)
continue
if self.thread_num > self.max_thread_num:
continue
target_conn_num = (1 - (connect_control.inactive_time()/(10*60))) * self.connection_pool_min_num
if self.new_conn_pool.qsize() > target_conn_num:
time.sleep(1)
continue
self.thread_num_lock.acquire()
self.thread_num += 1
self.thread_num_lock.release()
p = threading.Thread(target=self.connect_process)
p.start()
connect_start_num += 1
if connect_start_num > 10:
time.sleep(5)
connect_start_num = 0
def runJob(self):
while True: #not self.is_ip_enough() and self.searching_thread_count < 2:
if not connect_control.allow_connect():
break
try:
time.sleep(1)
ip_int = ip_range.get_ip()
ip_str = ip_utils.ip_num_to_string(ip_int)
if self.is_bad_ip(ip_str):
continue
result = check_ip.test_gws(ip_str)
if not result:
continue
if self.add_ip(ip_str, result.handshake_time, result.domain, result.server_type):
#logging.info("add %s CN:%s type:%s time:%d gws:%d ", ip_str,
# result.domain, result.server_type, result.handshake_time, len(self.gws_ip_list))
logging.info("check_ip add ip:%s time:%d", ip_str, result.handshake_time)
self.remove_slowest_ip()
self.save_ip_list()
except check_ip.HoneypotError as e:
self.report_bad_ip(ip_str)
connect_control.fall_into_honeypot()
break
except Exception as e:
logging.exception("google_ip.runJob fail:%s", e)
self.ncount_lock.acquire()
self.searching_thread_count -= 1
self.ncount_lock.release()
def connect_thread(self, sleep_time=0):
time.sleep(sleep_time)
try:
while self.new_conn_pool.qsize() < self.https_new_connect_num or \
self.new_conn_pool.qsize(only_h1=True) < 1:
if self.new_conn_pool.qsize() > self.connection_pool_max_num:
break
if not connect_control.allow_connect():
break
ip_str = google_ip.get_gws_ip()
if not ip_str:
xlog.warning("no enough ip")
time.sleep(60)
break
ssl_sock = self._create_ssl_connection((ip_str, 443))
if not ssl_sock:
continue
if self.ssl_timeout_cb and \
self.new_conn_pool.qsize() > self.connection_pool_max_num + 1 and \
self.new_conn_pool.qsize() > self.https_new_connect_num + 1 and \
self.new_conn_pool.qsize(only_h1=True) > 2:
self.ssl_timeout_cb(ssl_sock)
else:
self.new_conn_pool.put((ssl_sock.handshake_time, ssl_sock))
time.sleep(1)
finally:
self.thread_num_lock.acquire()
self.thread_num -= 1
self.thread_num_lock.release()
def create_connection_daemon(self):
connect_start_num = 0
while connect_control.keep_running:
time.sleep(0.1)
if not connect_control.allow_connect():
time.sleep(5)
continue
if self.thread_num > self.max_thread_num:
continue
target_conn_num = (1 - (connect_control.inactive_time()/(10*60))) * self.connection_pool_min_num
if self.new_conn_pool.qsize() > target_conn_num:
time.sleep(1)
continue
self.thread_num_lock.acquire()
self.thread_num += 1
self.thread_num_lock.release()
p = threading.Thread(target=self.connect_process)
p.start()
connect_start_num += 1
if connect_start_num > 10:
time.sleep(5)
connect_start_num = 0
def connect_thread(self, sleep_time=0):
time.sleep(sleep_time)
try:
while self.new_conn_pool.qsize() < self.connection_pool_min_num:
if self.new_conn_pool.qsize() >= self.connection_pool_min_num:
#xlog.debug("get enough conn")
break
ip_str = google_ip.get_gws_ip()
if not ip_str:
time.sleep(60)
xlog.warning("no enough ip")
break
port = 443
#logging.debug("create ssl conn %s", ip_str)
ssl_sock = self._create_ssl_connection( (ip_str, port) )
if ssl_sock:
ssl_sock.last_use_time = time.time()
self.new_conn_pool.put((ssl_sock.handshake_time, ssl_sock))
elif not connect_control.allow_connect():
break
time.sleep(1)
finally:
self.thread_num_lock.acquire()
self.thread_num -= 1
self.thread_num_lock.release()
def create_connection_worker(self, type="gae"):
try:
while connect_control.keep_running:
if type == "gae":
if (self.new_conn_pool.qsize() + self.gae_conn_pool.qsize()) >= self.connection_pool_min_num:
break
else:
if self.new_conn_pool.qsize() >= self.connection_pool_min_num:
break
ip_str = google_ip.get_gws_ip()
if not ip_str:
xlog.warning("no gws ip")
break
port = 443
#logging.debug("create ssl conn %s", ip_str)
ssl_sock = self._create_ssl_connection( (ip_str, port) )
if ssl_sock:
ssl_sock.last_use_time = time.time()
self.new_conn_pool.put((ssl_sock.handshake_time, ssl_sock))
elif not connect_control.allow_connect():
break
time.sleep(1)
finally:
self.thread_num_lock.acquire()
self.thread_num -= 1
self.thread_num_lock.release()
def connect_thread(self, sleep_time=0):
time.sleep(sleep_time)
try:
while self.new_conn_pool.qsize() < self.connection_pool_min_num:
if self.new_conn_pool.qsize() >= self.connection_pool_min_num:
#xlog.debug("get enough conn")
break
ip_str = google_ip.get_gws_ip()
if not ip_str:
xlog.warning("ip not enough")
break
port = 443
#logging.debug("create ssl conn %s", ip_str)
ssl_sock = self._create_ssl_connection((ip_str, port))
if ssl_sock:
ssl_sock.last_use_time = time.time()
self.new_conn_pool.put((ssl_sock.handshake_time, ssl_sock))
elif not connect_control.allow_connect():
break
time.sleep(1)
finally:
self.thread_num_lock.acquire()
self.thread_num -= 1
self.thread_num_lock.release()
def create_connection_worker(self, type="gae"):
try:
while connect_control.keep_running:
if type == "gae":
if (self.new_conn_pool.qsize() +
self.gae_conn_pool.qsize()
) >= self.connection_pool_min_num:
break
else:
if self.new_conn_pool.qsize(
) >= self.connection_pool_min_num:
break
ip_str = google_ip.get_gws_ip()
if not ip_str:
xlog.warning("no gws ip")
break
port = 443
#logging.debug("create ssl conn %s", ip_str)
connect_control.start_connect_register(True)
ssl_sock = self._create_ssl_connection((ip_str, port))
connect_control.end_connect_register(True)
if ssl_sock:
ssl_sock.last_use_time = time.time()
self.new_conn_pool.put((ssl_sock.handshake_time, ssl_sock))
elif not connect_control.allow_connect():
break
time.sleep(1)
finally:
self.thread_num_lock.acquire()
self.thread_num -= 1
self.thread_num_lock.release()
def connect_thread(self, sleep_time=0):
time.sleep(sleep_time)
try:
while connect_control.allow_connect():
#if self.new_conn_pool.qsize() > self.connection_pool_min:
if self.getter_num == 0 or self.new_conn_pool.qsize(
) > self.connection_pool_min:
break
ip_str = ip_manager.get_gws_ip()
if not ip_str:
xlog.warning("no enough ip")
time.sleep(10)
break
ssl_sock = self._create_ssl_connection((ip_str, 443))
if not ssl_sock:
continue
self.new_conn_pool.put((ssl_sock.handshake_time, ssl_sock))
time.sleep(1)
finally:
self.thread_num_lock.acquire()
self.thread_num -= 1
self.thread_num_lock.release()
def keep_connection_daemon(self):
while connect_control.keep_running:
if not connect_control.allow_connect():
time.sleep(5)
continue
if self.new_conn_pool.qsize() > self.connection_pool_min_num:
time.sleep(5)
continue
self.connect_process()
def keep_connection_daemon(self):
while connect_control.keep_running:
if not connect_control.allow_connect():
time.sleep(5)
continue
if self.new_conn_pool.qsize() > self.connection_pool_min_num:
time.sleep(5)
continue
self.connect_process()
def create_more_connection(self):
target_thread_num = min(self.max_thread_num, (self.connection_pool_min_num - self.conn_pool.qsize()))
while self.thread_num < target_thread_num and self.conn_pool.qsize() < self.connection_pool_min_num:
if not connect_control.allow_connect():
break
self.thread_num_lock.acquire()
self.thread_num += 1
self.thread_num_lock.release()
p = threading.Thread(target = self.connect_thread)
p.daemon = True
p.start()
time.sleep(0.3)
def create_more_connection_worker(self):
while connect_control.allow_connect() and \
self.thread_num < self.max_thread_num and \
self.new_conn_pool.qsize() < self.https_new_connect_num:
self.thread_num_lock.acquire()
self.thread_num += 1
self.thread_num_lock.release()
p = threading.Thread(target=self.connect_thread)
p.start()
time.sleep(0.5)
self.connecting_more_thread = None
def create_more_connection_worker(self):
while connect_control.allow_connect() and \
self.thread_num < self.max_connect_thread and \
(self.new_conn_pool.qsize() < self.connection_pool_min):
self.thread_num_lock.acquire()
self.thread_num += 1
self.thread_num_lock.release()
p = threading.Thread(target=self.connect_thread)
p.start()
time.sleep(0.5)
self.connecting_more_thread = None
def create_more_connection_worker(self):
need_conn_num = self.connection_pool_min_num - self.new_conn_pool.qsize()
target_thread_num = min(self.max_thread_num, need_conn_num)
while self.thread_num < target_thread_num and self.new_conn_pool.qsize() < self.connection_pool_min_num:
if not connect_control.allow_connect():
break
self.thread_num_lock.acquire()
self.thread_num += 1
self.thread_num_lock.release()
p = threading.Thread(target=self.connect_thread)
p.start()
time.sleep(0.3)
def _create_ssl_connection(self, ip_port):
if not connect_control.allow_connect():
time.sleep(10)
return False
sock = None
ssl_sock = None
ip = ip_port[0]
connect_control.start_connect_register(high_prior=True)
try:
ssl_sock = check_ip.connect_ssl(ip,
port=443,
timeout=self.timeout,
check_cert=True,
close_cb=google_ip.ssl_closed)
google_ip.update_ip(ip, ssl_sock.handshake_time)
xlog.debug("create_ssl update ip:%s time:%d h2:%d", ip,
ssl_sock.handshake_time, ssl_sock.h2)
connect_control.report_connect_success()
return ssl_sock
except check_ip.Cert_Exception as e:
xlog.debug("connect %s fail:%s ", ip, e)
google_ip.report_connect_fail(ip, force_remove=True)
if ssl_sock:
ssl_sock.close()
if sock:
sock.close()
except Exception as e:
xlog.debug("connect %s fail:%r", ip, e)
google_ip.report_connect_fail(ip)
connect_control.report_connect_fail()
if not check_local_network.IPv4.is_ok():
time.sleep(10)
else:
time.sleep(1)
if ssl_sock:
ssl_sock.close()
if sock:
sock.close()
finally:
connect_control.end_connect_register(high_prior=True)
def create_more_connection(self):
need_conn_num = self.connection_pool_min_num - self.new_conn_pool.qsize()
target_thread_num = min(self.max_thread_num, need_conn_num)
for i in range(0, target_thread_num):
if not connect_control.allow_connect():
xlog.warn("create more connect, control not allow")
time.sleep(10)
continue
if self.thread_num > self.max_thread_num:
break
self.thread_num_lock.acquire()
self.thread_num += 1
self.thread_num_lock.release()
p = threading.Thread(target=self.create_connection_worker)
p.start()
def create_more_connection(self, type="gae"):
if type == "gae":
need_conn_num = self.connection_pool_min_num - self.new_conn_pool.qsize() - self.gae_conn_pool.qsize()
else:
need_conn_num = self.connection_pool_min_num - self.new_conn_pool.qsize()
target_thread_num = min(self.max_thread_num, need_conn_num)
#while self.thread_num < self.max_thread_num and connect_control.keep_running:
for i in range(0, target_thread_num):
if not connect_control.allow_connect():
break
self.thread_num_lock.acquire()
self.thread_num += 1
self.thread_num_lock.release()
p = threading.Thread(target = self.create_connection_worker, args=(type,))
p.start()
def _create_ssl_connection(self, ip_port):
if not connect_control.allow_connect():
time.sleep(10)
return False
sock = None
ssl_sock = None
ip = ip_port[0]
connect_control.start_connect_register(high_prior=True)
try:
ssl_sock = check_ip.connect_ssl(ip, port=443, timeout=self.timeout, check_cert=True,
close_cb=google_ip.ssl_closed)
google_ip.update_ip(ip, ssl_sock.handshake_time)
xlog.debug("create_ssl update ip:%s time:%d h2:%d", ip, ssl_sock.handshake_time, ssl_sock.h2)
connect_control.report_connect_success()
return ssl_sock
except check_ip.Cert_Exception as e:
xlog.debug("connect %s fail:%s ", ip, e)
google_ip.report_connect_fail(ip, force_remove=True)
if ssl_sock:
ssl_sock.close()
if sock:
sock.close()
except Exception as e:
xlog.debug("connect %s fail:%r", ip, e)
google_ip.report_connect_fail(ip)
connect_control.report_connect_fail()
if not check_local_network.IPv4.is_ok():
time.sleep(10)
else:
time.sleep(1)
if ssl_sock:
ssl_sock.close()
if sock:
sock.close()
finally:
connect_control.end_connect_register(high_prior=True)
def create_more_connection(self):
need_conn_num = self.connection_pool_min_num - self.new_conn_pool.qsize(
)
target_thread_num = min(self.max_thread_num, need_conn_num)
for i in range(0, target_thread_num):
if not connect_control.allow_connect():
xlog.warn("create more connect, control not allow")
time.sleep(10)
continue
if self.thread_num > self.max_thread_num:
break
self.thread_num_lock.acquire()
self.thread_num += 1
self.thread_num_lock.release()
p = threading.Thread(target=self.create_connection_worker)
p.start()
def check_exist_ip(self):
self.ip_lock.acquire()
tmp_ip_list = [x for x in self.gws_ip_list]
self.ip_lock.release()
for ip_str in tmp_ip_list:
if not connect_control.allow_connect():
break
if self.is_bad_ip(ip_str):
self.report_connect_fail(ip_str, force_remove=True)
continue
try:
result = check_ip.test_gws(ip_str)
except check_ip.HoneypotError as e:
self.report_bad_ip(ip_str)
connect_control.fall_into_honeypot()
break
except Exception as e:
logging.exception("check_exist_ip fail:%s", e)
if not result:
if not check_ip.network_is_ok():
logging.warn(
"check_exist_ip network is fail, check your network connection."
)
return
logging.info("check_exist_ip fail ip:%s ", ip_str)
self.report_connect_fail(ip_str)
else:
self.update_ip(ip_str, result.handshake_time)
logging.info("check_exist_ip update ip:%s server:%s time:%d",
ip_str, result.server_type, result.handshake_time)
time.sleep(1)
self.save_ip_list()
def connect_thread(self, sleep_time=0):
time.sleep(sleep_time)
try:
while connect_control.allow_connect():
if self.new_conn_pool.qsize() > self.connection_pool_min:
break
ip_str = ip_manager.get_gws_ip()
if not ip_str:
xlog.warning("no enough ip")
time.sleep(60)
break
ssl_sock = self._create_ssl_connection( (ip_str, 443) )
if not ssl_sock:
continue
self.new_conn_pool.put((ssl_sock.handshake_time, ssl_sock))
time.sleep(1)
finally:
self.thread_num_lock.acquire()
self.thread_num -= 1
self.thread_num_lock.release()
def check_exist_ip(self):
self.ip_lock.acquire()
tmp_ip_list = [x for x in self.gws_ip_list]
self.ip_lock.release()
for ip_str in tmp_ip_list:
if not connect_control.allow_connect():
break
if self.is_bad_ip(ip_str):
self.report_connect_fail(ip_str, force_remove=True)
continue
try:
result = check_ip.test_gws(ip_str)
except check_ip.HoneypotError as e:
self.report_bad_ip(ip_str)
connect_control.fall_into_honeypot()
break
except Exception as e:
logging.exception("check_exist_ip fail:%s", e)
if not result:
if not self.network_is_ok():
logging.warn("check_exist_ip network is fail, check your network connection.")
return
logging.info("check_exist_ip fail ip:%s ", ip_str)
self.report_connect_fail(ip_str)
else:
self.update_ip(ip_str, result.handshake_time)
logging.info("check_exist_ip update ip:%s server:%s time:%d", ip_str, result.server_type, result.handshake_time)
time.sleep(1)
self.save_ip_list()
def create_more_connection(self):
target_thread_num = min(self.max_thread_num, (self.connection_pool_min_num - self.new_conn_pool.qsize()))
while self.thread_num < target_thread_num and self.new_conn_pool.qsize() < self.connection_pool_min_num and connect_control.allow_connect():
self.thread_num_lock.acquire()
self.thread_num += 1
self.thread_num_lock.release()
p = threading.Thread(target = self.connect_thread)
p.daemon = True
p.start()
time.sleep(0.3)
def _create_ssl_connection(self, ip_port):
if not connect_control.allow_connect():
return False
sock = None
ssl_sock = None
ip = ip_port[0]
connect_time = 0
handshake_time = 0
time_begin = time.time()
try:
if config.PROXY_ENABLE:
sock = socks.socksocket(socket.AF_INET if ':' not in
ip_port[0] else socket.AF_INET6)
else:
sock = socket.socket(socket.AF_INET if ':' not in
ip_port[0] else socket.AF_INET6)
# set reuseaddr option to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER,
struct.pack('ii', 1, 0))
# resize socket recv buffer 8K->32K to improve browser releated application performance
sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32 * 1024)
# disable negal algorithm to send http request quickly.
sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
# set a short timeout to trigger timeout retry more quickly.
sock.settimeout(self.timeout)
ssl_sock = SSLConnection(self.openssl_context, sock)
ssl_sock.set_connect_state()
# pick up the certificate
server_hostname = random_hostname()
if server_hostname and hasattr(ssl_sock, 'set_tlsext_host_name'):
ssl_sock.set_tlsext_host_name(server_hostname)
pass
ssl_sock.connect(ip_port)
time_connected = time.time()
ssl_sock.do_handshake()
time_handshaked = time.time()
connect_time = int((time_connected - time_begin) * 1000)
handshake_time = int((time_handshaked - time_connected) * 1000)
google_ip.update_ip(ip, handshake_time)
xlog.debug("create_ssl update ip:%s time:%d", ip, handshake_time)
# sometimes, we want to use raw tcp socket directly(select/epoll), so setattr it to ssl socket.
ssl_sock.ip = ip
ssl_sock.sock = sock
ssl_sock.create_time = time_begin
ssl_sock.handshake_time = handshake_time
ssl_sock.host = ''
def verify_SSL_certificate_issuer(ssl_sock):
cert = ssl_sock.get_peer_certificate()
if not cert:
#google_ip.report_bad_ip(ssl_sock.ip)
#connect_control.fall_into_honeypot()
raise socket.error(' certficate is none')
issuer_commonname = next(
(v for k, v in cert.get_issuer().get_components()
if k == 'CN'), '')
if not issuer_commonname.startswith('Google'):
google_ip.report_bad_ip(ssl_sock.ip)
connect_control.fall_into_honeypot()
raise socket.error(
' certficate is issued by %r, not Google' %
(issuer_commonname))
verify_SSL_certificate_issuer(ssl_sock)
connect_control.report_connect_success()
return ssl_sock
except Exception as e:
time_cost = time.time() - time_begin
xlog.debug("create_ssl %s fail:%s cost:%d h:%d", ip, e,
time_cost * 1000, handshake_time)
google_ip.report_connect_fail(ip)
connect_control.report_connect_fail()
if ssl_sock:
ssl_sock.close()
if sock:
sock.close()
return False
def _create_ssl_connection(self, ip_port):
if not connect_control.allow_connect():
return False
sock = None
ssl_sock = None
ip = ip_port[0]
connect_time = 0
handshake_time = 0
time_begin = time.time()
try:
if config.PROXY_ENABLE:
sock = socks.socksocket(socket.AF_INET if ":" not in ip_port[0] else socket.AF_INET6)
else:
sock = socket.socket(socket.AF_INET if ":" not in ip_port[0] else socket.AF_INET6)
# set reuseaddr option to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack("ii", 1, 0))
# resize socket recv buffer 8K->32K to improve browser releated application performance
sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32 * 1024)
# disable negal algorithm to send http request quickly.
sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
# set a short timeout to trigger timeout retry more quickly.
sock.settimeout(self.timeout)
ssl_sock = SSLConnection(self.openssl_context, sock)
ssl_sock.set_connect_state()
# pick up the certificate
server_hostname = random_hostname()
if server_hostname and hasattr(ssl_sock, "set_tlsext_host_name"):
ssl_sock.set_tlsext_host_name(server_hostname)
pass
ssl_sock.connect(ip_port)
time_connected = time.time()
ssl_sock.do_handshake()
time_handshaked = time.time()
connect_time = int((time_connected - time_begin) * 1000)
handshake_time = int((time_handshaked - time_connected) * 1000)
google_ip.update_ip(ip, handshake_time)
logging.debug("create_ssl update ip:%s time:%d", ip, handshake_time)
# sometimes, we want to use raw tcp socket directly(select/epoll), so setattr it to ssl socket.
ssl_sock.ip = ip
ssl_sock.sock = sock
ssl_sock.create_time = time_begin
ssl_sock.handshake_time = handshake_time
ssl_sock.host = ""
def verify_SSL_certificate_issuer(ssl_sock):
cert = ssl_sock.get_peer_certificate()
if not cert:
# google_ip.report_bad_ip(ssl_sock.ip)
# connect_control.fall_into_honeypot()
raise socket.error(" certficate is none")
issuer_commonname = next((v for k, v in cert.get_issuer().get_components() if k == "CN"), "")
if not issuer_commonname.startswith("Google"):
google_ip.report_bad_ip(ssl_sock.ip)
connect_control.fall_into_honeypot()
raise socket.error(" certficate is issued by %r, not Google" % (issuer_commonname))
verify_SSL_certificate_issuer(ssl_sock)
connect_control.report_connect_success()
return ssl_sock
except Exception as e:
time_cost = time.time() - time_begin
logging.debug("create_ssl %s fail:%s cost:%d h:%d", ip, e, time_cost * 1000, handshake_time)
google_ip.report_connect_fail(ip)
connect_control.report_connect_fail()
if ssl_sock:
ssl_sock.close()
if sock:
sock.close()
return False
def _create_ssl_connection(self, ip_port):
if not connect_control.allow_connect():
time.sleep(10)
return False
sock = None
ssl_sock = None
ip = ip_port[0]
connect_control.start_connect_register(high_prior=True)
connect_time = 0
handshake_time = 0
time_begin = time.time()
try:
if config.PROXY_ENABLE:
sock = socks.socksocket(socket.AF_INET if ':' not in ip else socket.AF_INET6)
else:
sock = socket.socket(socket.AF_INET if ':' not in ip else socket.AF_INET6)
# set reuseaddr option to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0))
# resize socket recv buffer 8K->32K to improve browser releated application performance
sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32*1024)
# disable negal algorithm to send http request quickly.
sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
# set a short timeout to trigger timeout retry more quickly.
sock.settimeout(self.timeout)
ssl_sock = SSLConnection(self.openssl_context, sock, ip, google_ip.ssl_closed)
ssl_sock.set_connect_state()
ssl_sock.connect(ip_port)
time_connected = time.time()
ssl_sock.do_handshake()
time_handshaked = time.time()
connect_time = int((time_connected - time_begin) * 1000)
handshake_time = int((time_handshaked - time_connected) * 1000)
google_ip.update_ip(ip, handshake_time)
xlog.debug("create_ssl update ip:%s time:%d", ip, handshake_time)
ssl_sock.fd = sock.fileno()
ssl_sock.create_time = time_begin
ssl_sock.received_size = 0
ssl_sock.load = 0
ssl_sock.handshake_time = handshake_time
ssl_sock.host = ''
def verify_SSL_certificate_issuer(ssl_sock):
cert = ssl_sock.get_peer_certificate()
if not cert:
#google_ip.report_bad_ip(ssl_sock.ip)
#connect_control.fall_into_honeypot()
raise socket.error(' certficate is none')
issuer_commonname = next((v for k, v in cert.get_issuer().get_components() if k == 'CN'), '')
if not issuer_commonname.startswith('Google'):
google_ip.report_connect_fail(ip, force_remove=True)
raise socket.error(' certficate is issued by %r, not Google' % ( issuer_commonname))
verify_SSL_certificate_issuer(ssl_sock)
connect_control.report_connect_success()
return ssl_sock
except Exception as e:
time_cost = time.time() - time_begin
if time_cost < self.timeout - 1:
xlog.debug("connect %s fail:%s cost:%d h:%d", ip, e, time_cost * 1000, handshake_time)
else:
xlog.debug("%s fail:%r", ip, e)
google_ip.report_connect_fail(ip)
connect_control.report_connect_fail()
if ssl_sock:
ssl_sock.close()
if sock:
sock.close()
return False
finally:
connect_control.end_connect_register(high_prior=True)
def _create_ssl_connection(self, ip_port):
if not connect_control.allow_connect():
time.sleep(10)
return False
sock = None
ssl_sock = None
ip = ip_port[0]
connect_control.start_connect_register(high_prior=True)
handshake_time = 0
time_begin = time.time()
try:
if config.PROXY_ENABLE:
sock = socks.socksocket(socket.AF_INET if ':' not in
ip else socket.AF_INET6)
else:
sock = socket.socket(socket.AF_INET if ':' not in
ip else socket.AF_INET6)
# set reuseaddr option to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER,
struct.pack('ii', 1, 0))
# resize socket recv buffer 8K->32K to improve browser releated application performance
sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 64 * 1024)
# disable negal algorithm to send http request quickly.
sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
# set a short timeout to trigger timeout retry more quickly.
sock.settimeout(self.connect_timeout)
ssl_sock = SSLConnection(self.openssl_context, sock, ip,
ip_manager.ssl_closed)
ssl_sock.set_connect_state()
host = random.choice(ns)
ssl_sock.set_tlsext_host_name(host)
ssl_sock.connect(ip_port)
time_connected = time.time()
ssl_sock.do_handshake()
time_handshaked = time.time()
def verify_SSL_certificate_issuer(ssl_sock):
cert = ssl_sock.get_peer_certificate()
if not cert:
#ip_manager.report_bad_ip(ssl_sock.ip)
#connect_control.fall_into_honeypot()
raise socket.error(' certficate is none')
issuer_commonname = next(
(v for k, v in cert.get_issuer().get_components()
if k == 'CN'), '')
if not issuer_commonname.startswith('COMODO'):
ip_manager.report_connect_fail(ip, force_remove=True)
raise socket.error(
' certficate is issued by %r, not COMODO' %
(issuer_commonname))
verify_SSL_certificate_issuer(ssl_sock)
handshake_time = int((time_handshaked - time_connected) * 1000)
try:
h2 = ssl_sock.get_alpn_proto_negotiated()
if h2 == "h2":
ssl_sock.h2 = True
# xlog.debug("ip:%s http/2", ip)
else:
ssl_sock.h2 = False
#xlog.deubg("alpn h2:%s", h2)
except:
if hasattr(ssl_sock._connection,
"protos") and ssl_sock._connection.protos == "h2":
ssl_sock.h2 = True
# xlog.debug("ip:%s http/2", ip)
else:
ssl_sock.h2 = False
# xlog.debug("ip:%s http/1.1", ip)
# ip_manager.update_ip(ip, handshake_time)
# handshake time is not the response time,
# cloudflare don't have global back-bond network like google.
# the reasonable response RTT time should be the HTTP test RTT.
xlog.debug("create_ssl update ip:%s time:%d h2:%d", ip,
handshake_time, ssl_sock.h2)
ssl_sock.fd = sock.fileno()
ssl_sock.create_time = time_begin
ssl_sock.last_use_time = time.time()
ssl_sock.received_size = 0
ssl_sock.load = 0
ssl_sock.handshake_time = handshake_time
ssl_sock.host = self.host
connect_control.report_connect_success()
return ssl_sock
except Exception as e:
time_cost = time.time() - time_begin
if time_cost < self.connect_timeout - 1:
xlog.debug("connect %s fail:%s cost:%d h:%d", ip, e,
time_cost * 1000, handshake_time)
else:
xlog.debug("%s fail:%r", ip, e)
ip_manager.report_connect_fail(ip)
connect_control.report_connect_fail()
if ssl_sock:
ssl_sock.close()
if sock:
sock.close()
return False
finally:
connect_control.end_connect_register(high_prior=True)
def _create_ssl_connection(self, ip_port):
if not connect_control.allow_connect():
time.sleep(10)
return False
sock = None
ssl_sock = None
ip = ip_port[0]
connect_time = 0
handshake_time = 0
time_begin = time.time()
try:
if config.PROXY_ENABLE:
sock = socks.socksocket(socket.AF_INET if ':' not in ip else socket.AF_INET6)
else:
sock = socket.socket(socket.AF_INET if ':' not in ip else socket.AF_INET6)
# set reuseaddr option to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0))
# resize socket recv buffer 8K->32K to improve browser releated application performance
sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32*1024)
# disable negal algorithm to send http request quickly.
sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
# set a short timeout to trigger timeout retry more quickly.
sock.settimeout(self.timeout)
ssl_sock = SSLConnection(self.openssl_context, sock, ip, google_ip.ssl_closed)
ssl_sock.set_connect_state()
ssl_sock.connect(ip_port)
time_connected = time.time()
ssl_sock.do_handshake()
time_handshaked = time.time()
connect_time = int((time_connected - time_begin) * 1000)
handshake_time = int((time_handshaked - time_connected) * 1000)
google_ip.update_ip(ip, handshake_time)
xlog.debug("create_ssl update ip:%s time:%d", ip, handshake_time)
ssl_sock.fd = sock.fileno()
ssl_sock.create_time = time_begin
ssl_sock.received_size = 0
ssl_sock.load = 0
ssl_sock.handshake_time = handshake_time
ssl_sock.host = ''
def verify_SSL_certificate_issuer(ssl_sock):
cert = ssl_sock.get_peer_certificate()
if not cert:
#google_ip.report_bad_ip(ssl_sock.ip)
#connect_control.fall_into_honeypot()
raise socket.error(' certficate is none')
issuer_commonname = next((v for k, v in cert.get_issuer().get_components() if k == 'CN'), '')
if not issuer_commonname.startswith('Google'):
google_ip.report_connect_fail(ip, force_remove=True)
raise socket.error(' certficate is issued by %r, not Google' % ( issuer_commonname))
verify_SSL_certificate_issuer(ssl_sock)
connect_control.report_connect_success()
return ssl_sock
except Exception as e:
time_cost = time.time() - time_begin
if time_cost < self.timeout - 1:
xlog.debug("connect %s fail:%s cost:%d h:%d", ip, e, time_cost * 1000, handshake_time)
else:
xlog.debug("%s fail:%r", ip, e)
google_ip.report_connect_fail(ip)
connect_control.report_connect_fail()
if ssl_sock:
ssl_sock.close()
if sock:
sock.close()
return False
def _create_ssl_connection(self, ip_port):
if not connect_control.allow_connect():
time.sleep(10)
return False
sock = None
ssl_sock = None
ip = ip_port[0]
connect_control.start_connect_register(high_prior=True)
handshake_time = 0
time_begin = time.time()
try:
if config.PROXY_ENABLE:
sock = socks.socksocket(socket.AF_INET if ':' not in
ip else socket.AF_INET6)
else:
sock = socket.socket(socket.AF_INET if ':' not in
ip else socket.AF_INET6)
# set reuseaddr option to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error
sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER,
struct.pack('ii', 1, 0))
# resize socket recv buffer 8K->32K to improve browser releated application performance
sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 64 * 1024)
# disable negal algorithm to send http request quickly.
sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True)
# set a short timeout to trigger timeout retry more quickly.
sock.settimeout(self.timeout)
ssl_sock = SSLConnection(self.openssl_context, sock, ip,
google_ip.ssl_closed)
ssl_sock.set_connect_state()
ssl_sock.connect(ip_port)
time_connected = time.time()
ssl_sock.do_handshake()
time_handshaked = time.time()
def verify_SSL_certificate_issuer(ssl_sock):
#cert = ssl_sock.get_peer_certificate()
#if not cert:
# #google_ip.report_bad_ip(ssl_sock.ip)
# #connect_control.fall_into_honeypot()
# raise socket.error(' certficate is none')
#issuer_commonname = next((v for k, v in cert.get_issuer().get_components() if k == 'CN'), '')
#if not issuer_commonname.startswith('Google'):
# google_ip.report_connect_fail(ip, force_remove=True)
# raise socket.error(' certficate is issued by %r, not Google' % ( issuer_commonname))
certs = ssl_sock.get_peer_cert_chain()
if not certs:
#google_ip.report_bad_ip(ssl_sock.ip)
#connect_control.fall_into_honeypot()
raise socket.error(' certficate is none')
if len(certs) < 3:
google_ip.report_connect_fail(ip, force_remove=True)
raise socket.error('No intermediate CA was found.')
if hasattr(OpenSSL.crypto, "dump_publickey"):
# old OpenSSL not support this function.
if OpenSSL.crypto.dump_publickey(
OpenSSL.crypto.FILETYPE_PEM,
certs[1].get_pubkey()) not in GoogleG23PKP:
google_ip.report_connect_fail(ip, force_remove=True)
raise socket.error(
'The intermediate CA is mismatching.')
issuer_commonname = next(
(v for k, v in certs[0].get_issuer().get_components()
if k == 'CN'), '')
if not issuer_commonname.startswith('Google'):
google_ip.report_connect_fail(ip, force_remove=True)
raise socket.error(
' certficate is issued by %r, not Google' %
(issuer_commonname))
verify_SSL_certificate_issuer(ssl_sock)
handshake_time = int((time_handshaked - time_connected) * 1000)
try:
h2 = ssl_sock.get_alpn_proto_negotiated()
if h2 == "h2":
ssl_sock.h2 = True
# xlog.debug("ip:%s http/2", ip)
else:
ssl_sock.h2 = False
#xlog.deubg("alpn h2:%s", h2)
except:
if hasattr(ssl_sock._connection,
"protos") and ssl_sock._connection.protos == "h2":
ssl_sock.h2 = True
# xlog.debug("ip:%s http/2", ip)
else:
ssl_sock.h2 = False
# xlog.debug("ip:%s http/1.1", ip)
google_ip.update_ip(ip, handshake_time)
xlog.debug("create_ssl update ip:%s time:%d h2:%d", ip,
handshake_time, ssl_sock.h2)
ssl_sock.fd = sock.fileno()
ssl_sock.create_time = time_begin
ssl_sock.last_use_time = time_begin
ssl_sock.received_size = 0
ssl_sock.load = 0
ssl_sock.handshake_time = handshake_time
ssl_sock.host = ''
connect_control.report_connect_success()
return ssl_sock
except Exception as e:
time_cost = time.time() - time_begin
if time_cost < self.timeout - 1:
xlog.debug("connect %s fail:%s cost:%d h:%d", ip, e,
time_cost * 1000, handshake_time)
else:
xlog.debug("%s fail:%r", ip, e)
google_ip.report_connect_fail(ip)
connect_control.report_connect_fail()
if ssl_sock:
ssl_sock.close()
if sock:
sock.close()
return False
finally:
connect_control.end_connect_register(high_prior=True)
