def remove_ip_process(self): try: while connect_control.keep_running: try: ip_str = self.to_remove_ip_list.get_nowait() except: break result = check_ip.test(ip_str) if result and result.appspot_ok: self.add_ip(ip_str, result.handshake_time, result.domain, result.server_type) xlog.debug("remove ip process, restore ip:%s", ip_str) continue if not check_ip.network_is_ok(): self.to_remove_ip_list.put(ip_str) xlog.warn( "network is unreachable. check your network connection." ) return xlog.info("real remove ip:%s ", ip_str) self.iplist_need_save = 1 finally: self.remove_ip_thread_num_lock.acquire() self.remove_ip_thread_num -= 1 self.remove_ip_thread_num_lock.release()
def report_connect_fail(self, ip_str, force_remove=False): self.ip_lock.acquire() try: time_now = time.time() if not ip_str in self.ip_dict: return self.ip_dict[ip_str]["links"] -= 1 # ignore if system network is disconnected. if not force_remove: if not check_ip.network_is_ok(): xlog.debug("report_connect_fail network fail") # connect_control.fall_into_honeypot() return fail_time = self.ip_dict[ip_str]["fail_time"] if not force_remove and time_now - fail_time < 1: xlog.debug("fail time too near") return # increase handshake_time to make it can be used in lower probability self.ip_dict[ip_str]["handshake_time"] += 300 if self.ip_dict[ip_str]["fail_times"] == 0: self.good_ip_num -= 1 self.ip_dict[ip_str]["fail_times"] += 1 self.append_ip_history(ip_str, "fail") self.ip_dict[ip_str]["fail_time"] = time_now if force_remove or self.ip_dict[ip_str]["fail_times"] >= 50: property = self.ip_dict[ip_str] server = property["server"] del self.ip_dict[ip_str] if "gws" in server and ip_str in self.gws_ip_list: self.gws_ip_list.remove(ip_str) if not force_remove: self.to_remove_ip_list.put(ip_str) self.try_remove_thread() xlog.info( "remove ip tmp:%s left amount:%d gws_num:%d", ip_str, len(self.ip_dict), len(self.gws_ip_list) ) else: xlog.info( "remove ip:%s left amount:%d gws_num:%d", ip_str, len(self.ip_dict), len(self.gws_ip_list) ) if self.good_ip_num > len(self.ip_dict): self.good_ip_num = len(self.ip_dict) self.iplist_need_save = 1 except Exception as e: xlog.exception("set_ip err:%s", e) finally: self.ip_lock.release() if not self.is_ip_enough(): self.search_more_google_ip()
def remove_ip_process(self): try: while connect_control.keep_running: try: ip_str = self.to_remove_ip_list.get_nowait() except: break result = check_ip.test(ip_str) if result and result.appspot_ok: self.add_ip(ip_str, result.handshake_time, result.domain, result.server_type) xlog.debug("remove ip process, restore ip:%s", ip_str) continue if not check_ip.network_is_ok(): self.to_remove_ip_list.put(ip_str) xlog.warn("network is unreachable. check your network connection.") return xlog.info("real remove ip:%s ", ip_str) self.iplist_need_save = 1 finally: self.remove_ip_thread_num_lock.acquire() self.remove_ip_thread_num -= 1 self.remove_ip_thread_num_lock.release()
def report_connect_fail(self, ip_str, force_remove=False): self.ip_lock.acquire() try: time_now = time.time() if not ip_str in self.ip_dict: return self.ip_dict[ip_str]['links'] -= 1 # ignore if system network is disconnected. if not force_remove: if not check_ip.network_is_ok(): xlog.debug("report_connect_fail network fail") #connect_control.fall_into_honeypot() return fail_time = self.ip_dict[ip_str]["fail_time"] if not force_remove and time_now - fail_time < 1: xlog.debug("fail time too near") return # increase handshake_time to make it can be used in lower probability self.ip_dict[ip_str]['handshake_time'] += 300 if self.ip_dict[ip_str]['fail_times'] == 0: self.good_ip_num -= 1 self.ip_dict[ip_str]['fail_times'] += 1 self.append_ip_history(ip_str, "fail") self.ip_dict[ip_str]["fail_time"] = time_now if force_remove or self.ip_dict[ip_str]['fail_times'] >= 50: property = self.ip_dict[ip_str] server = property['server'] del self.ip_dict[ip_str] if 'gws' in server and ip_str in self.gws_ip_list: self.gws_ip_list.remove(ip_str) if not force_remove: self.to_remove_ip_list.put(ip_str) self.try_remove_thread() xlog.info("remove ip tmp:%s left amount:%d gws_num:%d", ip_str, len(self.ip_dict), len(self.gws_ip_list)) else: xlog.info("remove ip:%s left amount:%d gws_num:%d", ip_str, len(self.ip_dict), len(self.gws_ip_list)) if self.good_ip_num > len(self.ip_dict): self.good_ip_num = len(self.ip_dict) self.iplist_need_save = 1 except Exception as e: xlog.exception("set_ip err:%s", e) finally: self.ip_lock.release() if not self.is_ip_enough(): self.search_more_google_ip()
def network_is_ok(self): if time.time() - self.network_fail_time < 3: return False if check_ip.network_is_ok(): return True self.network_fail_time = time.time() return False
def network_is_ok(self): if time.time() - self.network_fail_time < 3: return False if check_ip.network_is_ok(): logging.debug("network is ok") return True self.network_fail_time = time.time() logging.debug("network is fail") return False
def req_status_handler(self): if "user-agent" in self.headers.dict: user_agent = self.headers.dict["user-agent"] else: user_agent = "" good_ip_num = google_ip.good_ip_num if good_ip_num > len(google_ip.gws_ip_list): good_ip_num = len(google_ip.gws_ip_list) res_arr = { "sys_platform":"%s, %s" % (platform.machine(), platform.platform()), "os_system":platform.system(), "os_version":platform.version(), "os_release":platform.release(), "architecture":platform.architecture(), "os_detail":env_info.os_detail(), "language":self.get_os_language(), "browser":user_agent, "xxnet_version":self.xxnet_version(), "python_version": platform.python_version(), "proxy_listen":config.LISTEN_IP + ":" + str(config.LISTEN_PORT), "pac_url":config.pac_url, "use_ipv6":config.CONFIG.getint("google_ip", "use_ipv6"), "gae_appid":"|".join(config.GAE_APPIDS), "working_appid":"|".join(appid_manager.working_appid_list), "out_of_quota_appids":"|".join(appid_manager.out_of_quota_appids), "not_exist_appids":"|".join(appid_manager.not_exist_appids), "network_state":check_ip.network_is_ok(), "ip_num":len(google_ip.gws_ip_list), "good_ip_num":good_ip_num, "connected_link_new":len(https_manager.new_conn_pool.pool), "connected_link_used":len(https_manager.gae_conn_pool.pool), "scan_ip_thread_num":google_ip.searching_thread_count, "ip_quality": google_ip.ip_quality(), "block_stat":connect_control.block_stat(), "high_prior_connecting_num":connect_control.high_prior_connecting_num, "low_prior_connecting_num":connect_control.low_prior_connecting_num, "high_prior_lock":len(connect_control.high_prior_lock), "low_prior_lock":len(connect_control.low_prior_lock), } data = json.dumps(res_arr, indent=0, sort_keys=True) self.send_response('text/html', data)
def req_status_handler(self): if "user-agent" in self.headers.dict: user_agent = self.headers.dict["user-agent"] else: user_agent = "" good_ip_num = google_ip.good_ip_num if good_ip_num > len(google_ip.gws_ip_list): good_ip_num = len(google_ip.gws_ip_list) res_arr = { "sys_platform": "%s, %s" % (platform.machine(), platform.platform()), "os_system": platform.system(), "os_version": platform.version(), "os_release": platform.release(), "architecture": platform.architecture(), "os_detail": env_info.os_detail(), "language": self.get_os_language(), "browser": user_agent, "xxnet_version": self.xxnet_version(), "python_version": platform.python_version(), "proxy_listen": config.LISTEN_IP + ":" + str(config.LISTEN_PORT), "pac_url": config.pac_url, "use_ipv6": config.CONFIG.getint("google_ip", "use_ipv6"), "gae_appid": "|".join(config.GAE_APPIDS), "working_appid": "|".join(appid_manager.working_appid_list), "out_of_quota_appids": "|".join(appid_manager.out_of_quota_appids), "not_exist_appids": "|".join(appid_manager.not_exist_appids), "network_state": check_ip.network_is_ok(), "ip_num": len(google_ip.gws_ip_list), "good_ip_num": good_ip_num, "connected_link_new": len(https_manager.new_conn_pool.pool), "connected_link_used": len(https_manager.gae_conn_pool.pool), "scan_ip_thread_num": google_ip.searching_thread_count, "ip_quality": google_ip.ip_quality(), "block_stat": connect_control.block_stat(), "high_prior_connecting_num": connect_control.high_prior_connecting_num, "low_prior_connecting_num": connect_control.low_prior_connecting_num, "high_prior_lock": len(connect_control.high_prior_lock), "low_prior_lock": len(connect_control.low_prior_lock), } data = json.dumps(res_arr, indent=0, sort_keys=True) self.send_response('text/html', data)
def report_connect_fail(self, ip_str, force_remove=False): # ignore if system network is disconnected. if not force_remove: if not check_ip.network_is_ok(): logging.debug("report_connect_fail network fail") return self.ip_lock.acquire() try: if not ip_str in self.ip_dict: return fail_time = self.ip_dict[ip_str]["fail_time"] if not force_remove and time.time() - fail_time < 1: return # increase handshake_time to make it can be used in lower probability self.ip_dict[ip_str]['handshake_time'] += 200 self.ip_dict[ip_str]['timeout'] += 1 self.ip_dict[ip_str]['history'].append([time.time(), "fail"]) self.ip_dict[ip_str]["fail_time"] = time.time() if force_remove or self.ip_dict[ip_str]['timeout'] >= 50: property = self.ip_dict[ip_str] server = property['server'] del self.ip_dict[ip_str] if 'gws' in server and ip_str in self.gws_ip_list: self.gws_ip_list.remove(ip_str) logging.info("remove ip:%s left amount:%d gws_num:%d", ip_str, len(self.ip_dict), len(self.gws_ip_list)) if not force_remove: self.to_remove_ip_list.put(ip_str) self.try_remove_thread() self.iplist_need_save = 1 except Exception as e: logging.exception("set_ip err:%s", e) finally: self.ip_lock.release() if not self.is_ip_enough(): self.search_more_google_ip()
def report_connect_fail(self, ip_str, force_remove=False): # ignore if system network is disconnected. if not force_remove: if not check_ip.network_is_ok(): xlog.debug("report_connect_fail network fail") return self.ip_lock.acquire() try: if not ip_str in self.ip_dict: return fail_time = self.ip_dict[ip_str]["fail_time"] if not force_remove and time.time() - fail_time < 1: xlog.debug("fail time too near") return # increase handshake_time to make it can be used in lower probability self.ip_dict[ip_str]['handshake_time'] += 300 self.ip_dict[ip_str]['timeout'] += 1 self.ip_dict[ip_str]['history'].append([time.time(), "fail"]) self.ip_dict[ip_str]["fail_time"] = time.time() if force_remove or self.ip_dict[ip_str]['timeout'] >= 5: property = self.ip_dict[ip_str] server = property['server'] del self.ip_dict[ip_str] if 'gws' in server and ip_str in self.gws_ip_list: self.gws_ip_list.remove(ip_str) xlog.info("remove ip:%s left amount:%d gws_num:%d", ip_str, len(self.ip_dict), len(self.gws_ip_list)) if not force_remove: self.to_remove_ip_list.put(ip_str) self.try_remove_thread() self.iplist_need_save = 1 except Exception as e: xlog.exception("set_ip err:%s", e) finally: self.ip_lock.release() if not self.is_ip_enough(): self.search_more_google_ip()
def check_exist_ip(self): self.ip_lock.acquire() tmp_ip_list = [x for x in self.gws_ip_list] self.ip_lock.release() for ip_str in tmp_ip_list: if not connect_control.allow_connect(): break if self.is_bad_ip(ip_str): self.report_connect_fail(ip_str, force_remove=True) continue try: result = check_ip.test_gws(ip_str) except check_ip.HoneypotError as e: self.report_bad_ip(ip_str) connect_control.fall_into_honeypot() break except Exception as e: logging.exception("check_exist_ip fail:%s", e) if not result: if not check_ip.network_is_ok(): logging.warn( "check_exist_ip network is fail, check your network connection." ) return logging.info("check_exist_ip fail ip:%s ", ip_str) self.report_connect_fail(ip_str) else: self.update_ip(ip_str, result.handshake_time) logging.info("check_exist_ip update ip:%s server:%s time:%d", ip_str, result.server_type, result.handshake_time) time.sleep(1) self.save_ip_list()