def _create_ssl_connection(self, ip_port): if not connect_control.allow_connect(): time.sleep(10) return False ip = ip_port[0] port = ip_port[1] connect_control.start_connect_register(high_prior=True) time_begin = time.time() try: ssl_sock = check_ip.connect_ssl(ip, port=port, timeout=self.connect_timeout, on_close=ip_manager.ssl_closed) xlog.debug("create_ssl update ip:%s time:%d h2:%d sni:%s top:%s", ip, ssl_sock.handshake_time, ssl_sock.h2, ssl_sock.sni, ssl_sock.top_domain) ssl_sock.last_use_time = ssl_sock.create_time ssl_sock.received_size = 0 ssl_sock.load = 0 ssl_sock.host = self.sub + "." + ssl_sock.top_domain connect_control.report_connect_success() return ssl_sock except Exception as e: time_cost = time.time() - time_begin if time_cost < self.connect_timeout - 1: xlog.debug("connect %s fail:%s cost:%d ", ip, e, time_cost * 1000) else: xlog.debug("%s fail:%r", ip, e) ip_manager.report_connect_fail(ip) connect_control.report_connect_fail() return False finally: connect_control.end_connect_register(high_prior=True)
def connect_thread(self, sleep_time=0): time.sleep(sleep_time) try: while self.new_conn_pool.qsize() < self.https_new_connect_num or \ self.new_conn_pool.qsize(only_h1=True) < 1: if self.new_conn_pool.qsize() > self.connection_pool_max_num: break if not connect_control.allow_connect(): break ip_str = google_ip.get_gws_ip() if not ip_str: xlog.warning("no enough ip") time.sleep(60) break ssl_sock = self._create_ssl_connection( (ip_str, 443) ) if not ssl_sock: continue if self.ssl_timeout_cb and \ self.new_conn_pool.qsize() > self.connection_pool_max_num + 1 and \ self.new_conn_pool.qsize() > self.https_new_connect_num + 1 and \ self.new_conn_pool.qsize(only_h1=True) > 2: self.ssl_timeout_cb(ssl_sock) else: self.new_conn_pool.put((ssl_sock.handshake_time, ssl_sock)) time.sleep(1) finally: self.thread_num_lock.acquire() self.thread_num -= 1 self.thread_num_lock.release()
def runJob(self): while True: #not self.is_ip_enough() and self.searching_thread_count < 2: if not connect_control.allow_connect(): break try: time.sleep(1) ip_int = ip_range.get_ip() ip_str = ip_utils.ip_num_to_string(ip_int) if self.is_bad_ip(ip_str): continue result = check_ip.test_gws(ip_str) if not result: continue if self.add_ip(ip_str, result.handshake_time, result.domain, result.server_type): #logging.info("add %s CN:%s type:%s time:%d gws:%d ", ip_str, # result.domain, result.server_type, result.handshake_time, len(self.gws_ip_list)) logging.info("check_ip add ip:%s time:%d", ip_str, result.handshake_time) self.remove_slowest_ip() self.save_ip_list() except check_ip.HoneypotError as e: self.report_bad_ip(ip_str) connect_control.fall_into_honeypot() break except Exception as e: logging.exception("google_ip.runJob fail:%s", e) self.ncount_lock.acquire() self.searching_thread_count -= 1 self.ncount_lock.release()
def connect_thread(self, sleep_time=0): time.sleep(sleep_time) try: while self.new_conn_pool.qsize() < self.https_new_connect_num: ip_str = google_ip.get_gws_ip() if not ip_str: time.sleep(60) xlog.warning("no enough ip") break port = 443 #xlog.debug("create ssl conn %s", ip_str) ssl_sock = self._create_ssl_connection( (ip_str, port) ) if ssl_sock: ssl_sock.last_use_time = time.time() if self.new_conn_pool.qsize() >= self.connection_pool_max_num and self.ssl_timeout_cb: self.ssl_timeout_cb(ssl_sock) else: self.new_conn_pool.put((ssl_sock.handshake_time, ssl_sock)) elif not connect_control.allow_connect(): break time.sleep(1) finally: self.thread_num_lock.acquire() self.thread_num -= 1 self.thread_num_lock.release()
def create_connection_worker(self): try: while connect_control.keep_running: if self.new_conn_pool.qsize() >= self.connection_pool_min_num: #xlog.debug("get enough conn") break if self.thread_num > self.connection_pool_min_num - self.new_conn_pool.qsize(): break ip_str = google_ip.get_gws_ip() if not ip_str: xlog.warning("no gws ip") break port = 443 #logging.debug("create ssl conn %s", ip_str) connect_control.start_connect_register(True) ssl_sock = self._create_ssl_connection( (ip_str, port) ) connect_control.end_connect_register(True) if ssl_sock: ssl_sock.last_use_time = time.time() self.new_conn_pool.put((ssl_sock.handshake_time, ssl_sock)) elif not connect_control.allow_connect(): xlog.debug("create_connection_worker, control not allow") time.sleep(10) time.sleep(1) finally: self.thread_num_lock.acquire() self.thread_num -= 1 self.thread_num_lock.release()
def create_connection_daemon(self): connect_start_num = 0 while connect_control.keep_running: time.sleep(0.1) if not connect_control.allow_connect(): time.sleep(5) continue if self.thread_num > self.max_thread_num: continue target_conn_num = (1 - (connect_control.inactive_time()/(10*60))) * self.connection_pool_min_num if self.new_conn_pool.qsize() > target_conn_num: time.sleep(1) continue self.thread_num_lock.acquire() self.thread_num += 1 self.thread_num_lock.release() p = threading.Thread(target=self.connect_process) p.start() connect_start_num += 1 if connect_start_num > 10: time.sleep(5) connect_start_num = 0
def runJob(self): while True: #not self.is_ip_enough() and self.searching_thread_count < 2: if not connect_control.allow_connect(): break try: time.sleep(1) ip_int = ip_range.get_ip() ip_str = ip_utils.ip_num_to_string(ip_int) if self.is_bad_ip(ip_str): continue result = check_ip.test_gws(ip_str) if not result: continue if self.add_ip(ip_str, result.handshake_time, result.domain, result.server_type): #logging.info("add %s CN:%s type:%s time:%d gws:%d ", ip_str, # result.domain, result.server_type, result.handshake_time, len(self.gws_ip_list)) logging.info("check_ip add ip:%s time:%d", ip_str, result.handshake_time) self.remove_slowest_ip() self.save_ip_list() except check_ip.HoneypotError as e: self.report_bad_ip(ip_str) connect_control.fall_into_honeypot() break except Exception as e: logging.exception("google_ip.runJob fail:%s", e) self.ncount_lock.acquire() self.searching_thread_count -= 1 self.ncount_lock.release()
def connect_thread(self, sleep_time=0): time.sleep(sleep_time) try: while self.new_conn_pool.qsize() < self.https_new_connect_num or \ self.new_conn_pool.qsize(only_h1=True) < 1: if self.new_conn_pool.qsize() > self.connection_pool_max_num: break if not connect_control.allow_connect(): break ip_str = google_ip.get_gws_ip() if not ip_str: xlog.warning("no enough ip") time.sleep(60) break ssl_sock = self._create_ssl_connection((ip_str, 443)) if not ssl_sock: continue if self.ssl_timeout_cb and \ self.new_conn_pool.qsize() > self.connection_pool_max_num + 1 and \ self.new_conn_pool.qsize() > self.https_new_connect_num + 1 and \ self.new_conn_pool.qsize(only_h1=True) > 2: self.ssl_timeout_cb(ssl_sock) else: self.new_conn_pool.put((ssl_sock.handshake_time, ssl_sock)) time.sleep(1) finally: self.thread_num_lock.acquire() self.thread_num -= 1 self.thread_num_lock.release()
def create_connection_daemon(self): connect_start_num = 0 while connect_control.keep_running: time.sleep(0.1) if not connect_control.allow_connect(): time.sleep(5) continue if self.thread_num > self.max_thread_num: continue target_conn_num = (1 - (connect_control.inactive_time()/(10*60))) * self.connection_pool_min_num if self.new_conn_pool.qsize() > target_conn_num: time.sleep(1) continue self.thread_num_lock.acquire() self.thread_num += 1 self.thread_num_lock.release() p = threading.Thread(target=self.connect_process) p.start() connect_start_num += 1 if connect_start_num > 10: time.sleep(5) connect_start_num = 0
def connect_thread(self, sleep_time=0): time.sleep(sleep_time) try: while self.new_conn_pool.qsize() < self.connection_pool_min_num: if self.new_conn_pool.qsize() >= self.connection_pool_min_num: #xlog.debug("get enough conn") break ip_str = google_ip.get_gws_ip() if not ip_str: time.sleep(60) xlog.warning("no enough ip") break port = 443 #logging.debug("create ssl conn %s", ip_str) ssl_sock = self._create_ssl_connection( (ip_str, port) ) if ssl_sock: ssl_sock.last_use_time = time.time() self.new_conn_pool.put((ssl_sock.handshake_time, ssl_sock)) elif not connect_control.allow_connect(): break time.sleep(1) finally: self.thread_num_lock.acquire() self.thread_num -= 1 self.thread_num_lock.release()
def create_connection_worker(self, type="gae"): try: while connect_control.keep_running: if type == "gae": if (self.new_conn_pool.qsize() + self.gae_conn_pool.qsize()) >= self.connection_pool_min_num: break else: if self.new_conn_pool.qsize() >= self.connection_pool_min_num: break ip_str = google_ip.get_gws_ip() if not ip_str: xlog.warning("no gws ip") break port = 443 #logging.debug("create ssl conn %s", ip_str) ssl_sock = self._create_ssl_connection( (ip_str, port) ) if ssl_sock: ssl_sock.last_use_time = time.time() self.new_conn_pool.put((ssl_sock.handshake_time, ssl_sock)) elif not connect_control.allow_connect(): break time.sleep(1) finally: self.thread_num_lock.acquire() self.thread_num -= 1 self.thread_num_lock.release()
def connect_thread(self, sleep_time=0): time.sleep(sleep_time) try: while self.new_conn_pool.qsize() < self.connection_pool_min_num: if self.new_conn_pool.qsize() >= self.connection_pool_min_num: #xlog.debug("get enough conn") break ip_str = google_ip.get_gws_ip() if not ip_str: xlog.warning("ip not enough") break port = 443 #logging.debug("create ssl conn %s", ip_str) ssl_sock = self._create_ssl_connection((ip_str, port)) if ssl_sock: ssl_sock.last_use_time = time.time() self.new_conn_pool.put((ssl_sock.handshake_time, ssl_sock)) elif not connect_control.allow_connect(): break time.sleep(1) finally: self.thread_num_lock.acquire() self.thread_num -= 1 self.thread_num_lock.release()
def create_connection_worker(self, type="gae"): try: while connect_control.keep_running: if type == "gae": if (self.new_conn_pool.qsize() + self.gae_conn_pool.qsize() ) >= self.connection_pool_min_num: break else: if self.new_conn_pool.qsize( ) >= self.connection_pool_min_num: break ip_str = google_ip.get_gws_ip() if not ip_str: xlog.warning("no gws ip") break port = 443 #logging.debug("create ssl conn %s", ip_str) connect_control.start_connect_register(True) ssl_sock = self._create_ssl_connection((ip_str, port)) connect_control.end_connect_register(True) if ssl_sock: ssl_sock.last_use_time = time.time() self.new_conn_pool.put((ssl_sock.handshake_time, ssl_sock)) elif not connect_control.allow_connect(): break time.sleep(1) finally: self.thread_num_lock.acquire() self.thread_num -= 1 self.thread_num_lock.release()
def connect_thread(self, sleep_time=0): time.sleep(sleep_time) try: while connect_control.allow_connect(): #if self.new_conn_pool.qsize() > self.connection_pool_min: if self.getter_num == 0 or self.new_conn_pool.qsize( ) > self.connection_pool_min: break ip_str = ip_manager.get_gws_ip() if not ip_str: xlog.warning("no enough ip") time.sleep(10) break ssl_sock = self._create_ssl_connection((ip_str, 443)) if not ssl_sock: continue self.new_conn_pool.put((ssl_sock.handshake_time, ssl_sock)) time.sleep(1) finally: self.thread_num_lock.acquire() self.thread_num -= 1 self.thread_num_lock.release()
def keep_connection_daemon(self): while connect_control.keep_running: if not connect_control.allow_connect(): time.sleep(5) continue if self.new_conn_pool.qsize() > self.connection_pool_min_num: time.sleep(5) continue self.connect_process()
def keep_connection_daemon(self): while connect_control.keep_running: if not connect_control.allow_connect(): time.sleep(5) continue if self.new_conn_pool.qsize() > self.connection_pool_min_num: time.sleep(5) continue self.connect_process()
def create_more_connection(self): target_thread_num = min(self.max_thread_num, (self.connection_pool_min_num - self.conn_pool.qsize())) while self.thread_num < target_thread_num and self.conn_pool.qsize() < self.connection_pool_min_num: if not connect_control.allow_connect(): break self.thread_num_lock.acquire() self.thread_num += 1 self.thread_num_lock.release() p = threading.Thread(target = self.connect_thread) p.daemon = True p.start() time.sleep(0.3)
def create_more_connection_worker(self): while connect_control.allow_connect() and \ self.thread_num < self.max_thread_num and \ self.new_conn_pool.qsize() < self.https_new_connect_num: self.thread_num_lock.acquire() self.thread_num += 1 self.thread_num_lock.release() p = threading.Thread(target=self.connect_thread) p.start() time.sleep(0.5) self.connecting_more_thread = None
def create_more_connection_worker(self): while connect_control.allow_connect() and \ self.thread_num < self.max_connect_thread and \ (self.new_conn_pool.qsize() < self.connection_pool_min): self.thread_num_lock.acquire() self.thread_num += 1 self.thread_num_lock.release() p = threading.Thread(target=self.connect_thread) p.start() time.sleep(0.5) self.connecting_more_thread = None
def create_more_connection_worker(self): need_conn_num = self.connection_pool_min_num - self.new_conn_pool.qsize() target_thread_num = min(self.max_thread_num, need_conn_num) while self.thread_num < target_thread_num and self.new_conn_pool.qsize() < self.connection_pool_min_num: if not connect_control.allow_connect(): break self.thread_num_lock.acquire() self.thread_num += 1 self.thread_num_lock.release() p = threading.Thread(target=self.connect_thread) p.start() time.sleep(0.3)
def _create_ssl_connection(self, ip_port): if not connect_control.allow_connect(): time.sleep(10) return False sock = None ssl_sock = None ip = ip_port[0] connect_control.start_connect_register(high_prior=True) try: ssl_sock = check_ip.connect_ssl(ip, port=443, timeout=self.timeout, check_cert=True, close_cb=google_ip.ssl_closed) google_ip.update_ip(ip, ssl_sock.handshake_time) xlog.debug("create_ssl update ip:%s time:%d h2:%d", ip, ssl_sock.handshake_time, ssl_sock.h2) connect_control.report_connect_success() return ssl_sock except check_ip.Cert_Exception as e: xlog.debug("connect %s fail:%s ", ip, e) google_ip.report_connect_fail(ip, force_remove=True) if ssl_sock: ssl_sock.close() if sock: sock.close() except Exception as e: xlog.debug("connect %s fail:%r", ip, e) google_ip.report_connect_fail(ip) connect_control.report_connect_fail() if not check_local_network.IPv4.is_ok(): time.sleep(10) else: time.sleep(1) if ssl_sock: ssl_sock.close() if sock: sock.close() finally: connect_control.end_connect_register(high_prior=True)
def create_more_connection(self): need_conn_num = self.connection_pool_min_num - self.new_conn_pool.qsize() target_thread_num = min(self.max_thread_num, need_conn_num) for i in range(0, target_thread_num): if not connect_control.allow_connect(): xlog.warn("create more connect, control not allow") time.sleep(10) continue if self.thread_num > self.max_thread_num: break self.thread_num_lock.acquire() self.thread_num += 1 self.thread_num_lock.release() p = threading.Thread(target=self.create_connection_worker) p.start()
def create_more_connection(self, type="gae"): if type == "gae": need_conn_num = self.connection_pool_min_num - self.new_conn_pool.qsize() - self.gae_conn_pool.qsize() else: need_conn_num = self.connection_pool_min_num - self.new_conn_pool.qsize() target_thread_num = min(self.max_thread_num, need_conn_num) #while self.thread_num < self.max_thread_num and connect_control.keep_running: for i in range(0, target_thread_num): if not connect_control.allow_connect(): break self.thread_num_lock.acquire() self.thread_num += 1 self.thread_num_lock.release() p = threading.Thread(target = self.create_connection_worker, args=(type,)) p.start()
def _create_ssl_connection(self, ip_port): if not connect_control.allow_connect(): time.sleep(10) return False sock = None ssl_sock = None ip = ip_port[0] connect_control.start_connect_register(high_prior=True) try: ssl_sock = check_ip.connect_ssl(ip, port=443, timeout=self.timeout, check_cert=True, close_cb=google_ip.ssl_closed) google_ip.update_ip(ip, ssl_sock.handshake_time) xlog.debug("create_ssl update ip:%s time:%d h2:%d", ip, ssl_sock.handshake_time, ssl_sock.h2) connect_control.report_connect_success() return ssl_sock except check_ip.Cert_Exception as e: xlog.debug("connect %s fail:%s ", ip, e) google_ip.report_connect_fail(ip, force_remove=True) if ssl_sock: ssl_sock.close() if sock: sock.close() except Exception as e: xlog.debug("connect %s fail:%r", ip, e) google_ip.report_connect_fail(ip) connect_control.report_connect_fail() if not check_local_network.IPv4.is_ok(): time.sleep(10) else: time.sleep(1) if ssl_sock: ssl_sock.close() if sock: sock.close() finally: connect_control.end_connect_register(high_prior=True)
def create_more_connection(self): need_conn_num = self.connection_pool_min_num - self.new_conn_pool.qsize( ) target_thread_num = min(self.max_thread_num, need_conn_num) for i in range(0, target_thread_num): if not connect_control.allow_connect(): xlog.warn("create more connect, control not allow") time.sleep(10) continue if self.thread_num > self.max_thread_num: break self.thread_num_lock.acquire() self.thread_num += 1 self.thread_num_lock.release() p = threading.Thread(target=self.create_connection_worker) p.start()
def check_exist_ip(self): self.ip_lock.acquire() tmp_ip_list = [x for x in self.gws_ip_list] self.ip_lock.release() for ip_str in tmp_ip_list: if not connect_control.allow_connect(): break if self.is_bad_ip(ip_str): self.report_connect_fail(ip_str, force_remove=True) continue try: result = check_ip.test_gws(ip_str) except check_ip.HoneypotError as e: self.report_bad_ip(ip_str) connect_control.fall_into_honeypot() break except Exception as e: logging.exception("check_exist_ip fail:%s", e) if not result: if not check_ip.network_is_ok(): logging.warn( "check_exist_ip network is fail, check your network connection." ) return logging.info("check_exist_ip fail ip:%s ", ip_str) self.report_connect_fail(ip_str) else: self.update_ip(ip_str, result.handshake_time) logging.info("check_exist_ip update ip:%s server:%s time:%d", ip_str, result.server_type, result.handshake_time) time.sleep(1) self.save_ip_list()
def connect_thread(self, sleep_time=0): time.sleep(sleep_time) try: while connect_control.allow_connect(): if self.new_conn_pool.qsize() > self.connection_pool_min: break ip_str = ip_manager.get_gws_ip() if not ip_str: xlog.warning("no enough ip") time.sleep(60) break ssl_sock = self._create_ssl_connection( (ip_str, 443) ) if not ssl_sock: continue self.new_conn_pool.put((ssl_sock.handshake_time, ssl_sock)) time.sleep(1) finally: self.thread_num_lock.acquire() self.thread_num -= 1 self.thread_num_lock.release()
def check_exist_ip(self): self.ip_lock.acquire() tmp_ip_list = [x for x in self.gws_ip_list] self.ip_lock.release() for ip_str in tmp_ip_list: if not connect_control.allow_connect(): break if self.is_bad_ip(ip_str): self.report_connect_fail(ip_str, force_remove=True) continue try: result = check_ip.test_gws(ip_str) except check_ip.HoneypotError as e: self.report_bad_ip(ip_str) connect_control.fall_into_honeypot() break except Exception as e: logging.exception("check_exist_ip fail:%s", e) if not result: if not self.network_is_ok(): logging.warn("check_exist_ip network is fail, check your network connection.") return logging.info("check_exist_ip fail ip:%s ", ip_str) self.report_connect_fail(ip_str) else: self.update_ip(ip_str, result.handshake_time) logging.info("check_exist_ip update ip:%s server:%s time:%d", ip_str, result.server_type, result.handshake_time) time.sleep(1) self.save_ip_list()
def create_more_connection(self): target_thread_num = min(self.max_thread_num, (self.connection_pool_min_num - self.new_conn_pool.qsize())) while self.thread_num < target_thread_num and self.new_conn_pool.qsize() < self.connection_pool_min_num and connect_control.allow_connect(): self.thread_num_lock.acquire() self.thread_num += 1 self.thread_num_lock.release() p = threading.Thread(target = self.connect_thread) p.daemon = True p.start() time.sleep(0.3)
def _create_ssl_connection(self, ip_port): if not connect_control.allow_connect(): return False sock = None ssl_sock = None ip = ip_port[0] connect_time = 0 handshake_time = 0 time_begin = time.time() try: if config.PROXY_ENABLE: sock = socks.socksocket(socket.AF_INET if ':' not in ip_port[0] else socket.AF_INET6) else: sock = socket.socket(socket.AF_INET if ':' not in ip_port[0] else socket.AF_INET6) # set reuseaddr option to avoid 10048 socket error sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0)) # resize socket recv buffer 8K->32K to improve browser releated application performance sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32 * 1024) # disable negal algorithm to send http request quickly. sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True) # set a short timeout to trigger timeout retry more quickly. sock.settimeout(self.timeout) ssl_sock = SSLConnection(self.openssl_context, sock) ssl_sock.set_connect_state() # pick up the certificate server_hostname = random_hostname() if server_hostname and hasattr(ssl_sock, 'set_tlsext_host_name'): ssl_sock.set_tlsext_host_name(server_hostname) pass ssl_sock.connect(ip_port) time_connected = time.time() ssl_sock.do_handshake() time_handshaked = time.time() connect_time = int((time_connected - time_begin) * 1000) handshake_time = int((time_handshaked - time_connected) * 1000) google_ip.update_ip(ip, handshake_time) xlog.debug("create_ssl update ip:%s time:%d", ip, handshake_time) # sometimes, we want to use raw tcp socket directly(select/epoll), so setattr it to ssl socket. ssl_sock.ip = ip ssl_sock.sock = sock ssl_sock.create_time = time_begin ssl_sock.handshake_time = handshake_time ssl_sock.host = '' def verify_SSL_certificate_issuer(ssl_sock): cert = ssl_sock.get_peer_certificate() if not cert: #google_ip.report_bad_ip(ssl_sock.ip) #connect_control.fall_into_honeypot() raise socket.error(' certficate is none') issuer_commonname = next( (v for k, v in cert.get_issuer().get_components() if k == 'CN'), '') if not issuer_commonname.startswith('Google'): google_ip.report_bad_ip(ssl_sock.ip) connect_control.fall_into_honeypot() raise socket.error( ' certficate is issued by %r, not Google' % (issuer_commonname)) verify_SSL_certificate_issuer(ssl_sock) connect_control.report_connect_success() return ssl_sock except Exception as e: time_cost = time.time() - time_begin xlog.debug("create_ssl %s fail:%s cost:%d h:%d", ip, e, time_cost * 1000, handshake_time) google_ip.report_connect_fail(ip) connect_control.report_connect_fail() if ssl_sock: ssl_sock.close() if sock: sock.close() return False
def _create_ssl_connection(self, ip_port): if not connect_control.allow_connect(): return False sock = None ssl_sock = None ip = ip_port[0] connect_time = 0 handshake_time = 0 time_begin = time.time() try: if config.PROXY_ENABLE: sock = socks.socksocket(socket.AF_INET if ":" not in ip_port[0] else socket.AF_INET6) else: sock = socket.socket(socket.AF_INET if ":" not in ip_port[0] else socket.AF_INET6) # set reuseaddr option to avoid 10048 socket error sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack("ii", 1, 0)) # resize socket recv buffer 8K->32K to improve browser releated application performance sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32 * 1024) # disable negal algorithm to send http request quickly. sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True) # set a short timeout to trigger timeout retry more quickly. sock.settimeout(self.timeout) ssl_sock = SSLConnection(self.openssl_context, sock) ssl_sock.set_connect_state() # pick up the certificate server_hostname = random_hostname() if server_hostname and hasattr(ssl_sock, "set_tlsext_host_name"): ssl_sock.set_tlsext_host_name(server_hostname) pass ssl_sock.connect(ip_port) time_connected = time.time() ssl_sock.do_handshake() time_handshaked = time.time() connect_time = int((time_connected - time_begin) * 1000) handshake_time = int((time_handshaked - time_connected) * 1000) google_ip.update_ip(ip, handshake_time) logging.debug("create_ssl update ip:%s time:%d", ip, handshake_time) # sometimes, we want to use raw tcp socket directly(select/epoll), so setattr it to ssl socket. ssl_sock.ip = ip ssl_sock.sock = sock ssl_sock.create_time = time_begin ssl_sock.handshake_time = handshake_time ssl_sock.host = "" def verify_SSL_certificate_issuer(ssl_sock): cert = ssl_sock.get_peer_certificate() if not cert: # google_ip.report_bad_ip(ssl_sock.ip) # connect_control.fall_into_honeypot() raise socket.error(" certficate is none") issuer_commonname = next((v for k, v in cert.get_issuer().get_components() if k == "CN"), "") if not issuer_commonname.startswith("Google"): google_ip.report_bad_ip(ssl_sock.ip) connect_control.fall_into_honeypot() raise socket.error(" certficate is issued by %r, not Google" % (issuer_commonname)) verify_SSL_certificate_issuer(ssl_sock) connect_control.report_connect_success() return ssl_sock except Exception as e: time_cost = time.time() - time_begin logging.debug("create_ssl %s fail:%s cost:%d h:%d", ip, e, time_cost * 1000, handshake_time) google_ip.report_connect_fail(ip) connect_control.report_connect_fail() if ssl_sock: ssl_sock.close() if sock: sock.close() return False
def _create_ssl_connection(self, ip_port): if not connect_control.allow_connect(): time.sleep(10) return False sock = None ssl_sock = None ip = ip_port[0] connect_control.start_connect_register(high_prior=True) connect_time = 0 handshake_time = 0 time_begin = time.time() try: if config.PROXY_ENABLE: sock = socks.socksocket(socket.AF_INET if ':' not in ip else socket.AF_INET6) else: sock = socket.socket(socket.AF_INET if ':' not in ip else socket.AF_INET6) # set reuseaddr option to avoid 10048 socket error sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0)) # resize socket recv buffer 8K->32K to improve browser releated application performance sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32*1024) # disable negal algorithm to send http request quickly. sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True) # set a short timeout to trigger timeout retry more quickly. sock.settimeout(self.timeout) ssl_sock = SSLConnection(self.openssl_context, sock, ip, google_ip.ssl_closed) ssl_sock.set_connect_state() ssl_sock.connect(ip_port) time_connected = time.time() ssl_sock.do_handshake() time_handshaked = time.time() connect_time = int((time_connected - time_begin) * 1000) handshake_time = int((time_handshaked - time_connected) * 1000) google_ip.update_ip(ip, handshake_time) xlog.debug("create_ssl update ip:%s time:%d", ip, handshake_time) ssl_sock.fd = sock.fileno() ssl_sock.create_time = time_begin ssl_sock.received_size = 0 ssl_sock.load = 0 ssl_sock.handshake_time = handshake_time ssl_sock.host = '' def verify_SSL_certificate_issuer(ssl_sock): cert = ssl_sock.get_peer_certificate() if not cert: #google_ip.report_bad_ip(ssl_sock.ip) #connect_control.fall_into_honeypot() raise socket.error(' certficate is none') issuer_commonname = next((v for k, v in cert.get_issuer().get_components() if k == 'CN'), '') if not issuer_commonname.startswith('Google'): google_ip.report_connect_fail(ip, force_remove=True) raise socket.error(' certficate is issued by %r, not Google' % ( issuer_commonname)) verify_SSL_certificate_issuer(ssl_sock) connect_control.report_connect_success() return ssl_sock except Exception as e: time_cost = time.time() - time_begin if time_cost < self.timeout - 1: xlog.debug("connect %s fail:%s cost:%d h:%d", ip, e, time_cost * 1000, handshake_time) else: xlog.debug("%s fail:%r", ip, e) google_ip.report_connect_fail(ip) connect_control.report_connect_fail() if ssl_sock: ssl_sock.close() if sock: sock.close() return False finally: connect_control.end_connect_register(high_prior=True)
def _create_ssl_connection(self, ip_port): if not connect_control.allow_connect(): time.sleep(10) return False sock = None ssl_sock = None ip = ip_port[0] connect_control.start_connect_register(high_prior=True) handshake_time = 0 time_begin = time.time() try: if config.PROXY_ENABLE: sock = socks.socksocket(socket.AF_INET if ':' not in ip else socket.AF_INET6) else: sock = socket.socket(socket.AF_INET if ':' not in ip else socket.AF_INET6) # set reuseaddr option to avoid 10048 socket error sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0)) # resize socket recv buffer 8K->32K to improve browser releated application performance sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 64 * 1024) # disable negal algorithm to send http request quickly. sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True) # set a short timeout to trigger timeout retry more quickly. sock.settimeout(self.connect_timeout) ssl_sock = SSLConnection(self.openssl_context, sock, ip, ip_manager.ssl_closed) ssl_sock.set_connect_state() host = random.choice(ns) ssl_sock.set_tlsext_host_name(host) ssl_sock.connect(ip_port) time_connected = time.time() ssl_sock.do_handshake() time_handshaked = time.time() def verify_SSL_certificate_issuer(ssl_sock): cert = ssl_sock.get_peer_certificate() if not cert: #ip_manager.report_bad_ip(ssl_sock.ip) #connect_control.fall_into_honeypot() raise socket.error(' certficate is none') issuer_commonname = next( (v for k, v in cert.get_issuer().get_components() if k == 'CN'), '') if not issuer_commonname.startswith('COMODO'): ip_manager.report_connect_fail(ip, force_remove=True) raise socket.error( ' certficate is issued by %r, not COMODO' % (issuer_commonname)) verify_SSL_certificate_issuer(ssl_sock) handshake_time = int((time_handshaked - time_connected) * 1000) try: h2 = ssl_sock.get_alpn_proto_negotiated() if h2 == "h2": ssl_sock.h2 = True # xlog.debug("ip:%s http/2", ip) else: ssl_sock.h2 = False #xlog.deubg("alpn h2:%s", h2) except: if hasattr(ssl_sock._connection, "protos") and ssl_sock._connection.protos == "h2": ssl_sock.h2 = True # xlog.debug("ip:%s http/2", ip) else: ssl_sock.h2 = False # xlog.debug("ip:%s http/1.1", ip) # ip_manager.update_ip(ip, handshake_time) # handshake time is not the response time, # cloudflare don't have global back-bond network like google. # the reasonable response RTT time should be the HTTP test RTT. xlog.debug("create_ssl update ip:%s time:%d h2:%d", ip, handshake_time, ssl_sock.h2) ssl_sock.fd = sock.fileno() ssl_sock.create_time = time_begin ssl_sock.last_use_time = time.time() ssl_sock.received_size = 0 ssl_sock.load = 0 ssl_sock.handshake_time = handshake_time ssl_sock.host = self.host connect_control.report_connect_success() return ssl_sock except Exception as e: time_cost = time.time() - time_begin if time_cost < self.connect_timeout - 1: xlog.debug("connect %s fail:%s cost:%d h:%d", ip, e, time_cost * 1000, handshake_time) else: xlog.debug("%s fail:%r", ip, e) ip_manager.report_connect_fail(ip) connect_control.report_connect_fail() if ssl_sock: ssl_sock.close() if sock: sock.close() return False finally: connect_control.end_connect_register(high_prior=True)
def _create_ssl_connection(self, ip_port): if not connect_control.allow_connect(): time.sleep(10) return False sock = None ssl_sock = None ip = ip_port[0] connect_time = 0 handshake_time = 0 time_begin = time.time() try: if config.PROXY_ENABLE: sock = socks.socksocket(socket.AF_INET if ':' not in ip else socket.AF_INET6) else: sock = socket.socket(socket.AF_INET if ':' not in ip else socket.AF_INET6) # set reuseaddr option to avoid 10048 socket error sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0)) # resize socket recv buffer 8K->32K to improve browser releated application performance sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 32*1024) # disable negal algorithm to send http request quickly. sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True) # set a short timeout to trigger timeout retry more quickly. sock.settimeout(self.timeout) ssl_sock = SSLConnection(self.openssl_context, sock, ip, google_ip.ssl_closed) ssl_sock.set_connect_state() ssl_sock.connect(ip_port) time_connected = time.time() ssl_sock.do_handshake() time_handshaked = time.time() connect_time = int((time_connected - time_begin) * 1000) handshake_time = int((time_handshaked - time_connected) * 1000) google_ip.update_ip(ip, handshake_time) xlog.debug("create_ssl update ip:%s time:%d", ip, handshake_time) ssl_sock.fd = sock.fileno() ssl_sock.create_time = time_begin ssl_sock.received_size = 0 ssl_sock.load = 0 ssl_sock.handshake_time = handshake_time ssl_sock.host = '' def verify_SSL_certificate_issuer(ssl_sock): cert = ssl_sock.get_peer_certificate() if not cert: #google_ip.report_bad_ip(ssl_sock.ip) #connect_control.fall_into_honeypot() raise socket.error(' certficate is none') issuer_commonname = next((v for k, v in cert.get_issuer().get_components() if k == 'CN'), '') if not issuer_commonname.startswith('Google'): google_ip.report_connect_fail(ip, force_remove=True) raise socket.error(' certficate is issued by %r, not Google' % ( issuer_commonname)) verify_SSL_certificate_issuer(ssl_sock) connect_control.report_connect_success() return ssl_sock except Exception as e: time_cost = time.time() - time_begin if time_cost < self.timeout - 1: xlog.debug("connect %s fail:%s cost:%d h:%d", ip, e, time_cost * 1000, handshake_time) else: xlog.debug("%s fail:%r", ip, e) google_ip.report_connect_fail(ip) connect_control.report_connect_fail() if ssl_sock: ssl_sock.close() if sock: sock.close() return False
def _create_ssl_connection(self, ip_port): if not connect_control.allow_connect(): time.sleep(10) return False sock = None ssl_sock = None ip = ip_port[0] connect_control.start_connect_register(high_prior=True) handshake_time = 0 time_begin = time.time() try: if config.PROXY_ENABLE: sock = socks.socksocket(socket.AF_INET if ':' not in ip else socket.AF_INET6) else: sock = socket.socket(socket.AF_INET if ':' not in ip else socket.AF_INET6) # set reuseaddr option to avoid 10048 socket error sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # set struct linger{l_onoff=1,l_linger=0} to avoid 10048 socket error sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0)) # resize socket recv buffer 8K->32K to improve browser releated application performance sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 64 * 1024) # disable negal algorithm to send http request quickly. sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, True) # set a short timeout to trigger timeout retry more quickly. sock.settimeout(self.timeout) ssl_sock = SSLConnection(self.openssl_context, sock, ip, google_ip.ssl_closed) ssl_sock.set_connect_state() ssl_sock.connect(ip_port) time_connected = time.time() ssl_sock.do_handshake() time_handshaked = time.time() def verify_SSL_certificate_issuer(ssl_sock): #cert = ssl_sock.get_peer_certificate() #if not cert: # #google_ip.report_bad_ip(ssl_sock.ip) # #connect_control.fall_into_honeypot() # raise socket.error(' certficate is none') #issuer_commonname = next((v for k, v in cert.get_issuer().get_components() if k == 'CN'), '') #if not issuer_commonname.startswith('Google'): # google_ip.report_connect_fail(ip, force_remove=True) # raise socket.error(' certficate is issued by %r, not Google' % ( issuer_commonname)) certs = ssl_sock.get_peer_cert_chain() if not certs: #google_ip.report_bad_ip(ssl_sock.ip) #connect_control.fall_into_honeypot() raise socket.error(' certficate is none') if len(certs) < 3: google_ip.report_connect_fail(ip, force_remove=True) raise socket.error('No intermediate CA was found.') if hasattr(OpenSSL.crypto, "dump_publickey"): # old OpenSSL not support this function. if OpenSSL.crypto.dump_publickey( OpenSSL.crypto.FILETYPE_PEM, certs[1].get_pubkey()) not in GoogleG23PKP: google_ip.report_connect_fail(ip, force_remove=True) raise socket.error( 'The intermediate CA is mismatching.') issuer_commonname = next( (v for k, v in certs[0].get_issuer().get_components() if k == 'CN'), '') if not issuer_commonname.startswith('Google'): google_ip.report_connect_fail(ip, force_remove=True) raise socket.error( ' certficate is issued by %r, not Google' % (issuer_commonname)) verify_SSL_certificate_issuer(ssl_sock) handshake_time = int((time_handshaked - time_connected) * 1000) try: h2 = ssl_sock.get_alpn_proto_negotiated() if h2 == "h2": ssl_sock.h2 = True # xlog.debug("ip:%s http/2", ip) else: ssl_sock.h2 = False #xlog.deubg("alpn h2:%s", h2) except: if hasattr(ssl_sock._connection, "protos") and ssl_sock._connection.protos == "h2": ssl_sock.h2 = True # xlog.debug("ip:%s http/2", ip) else: ssl_sock.h2 = False # xlog.debug("ip:%s http/1.1", ip) google_ip.update_ip(ip, handshake_time) xlog.debug("create_ssl update ip:%s time:%d h2:%d", ip, handshake_time, ssl_sock.h2) ssl_sock.fd = sock.fileno() ssl_sock.create_time = time_begin ssl_sock.last_use_time = time_begin ssl_sock.received_size = 0 ssl_sock.load = 0 ssl_sock.handshake_time = handshake_time ssl_sock.host = '' connect_control.report_connect_success() return ssl_sock except Exception as e: time_cost = time.time() - time_begin if time_cost < self.timeout - 1: xlog.debug("connect %s fail:%s cost:%d h:%d", ip, e, time_cost * 1000, handshake_time) else: xlog.debug("%s fail:%r", ip, e) google_ip.report_connect_fail(ip) connect_control.report_connect_fail() if ssl_sock: ssl_sock.close() if sock: sock.close() return False finally: connect_control.end_connect_register(high_prior=True)