def add_new_vmess( v2ray_url, crawl_id: int = 0, interval: int = global_variable.get_conf_int("INTERVAL", default=1800), ) -> bool: try: if v2ray_url == "": return False # 已经存在了,就不管了 data = (global_variable.get_db().query(SubscribeVmss).filter( SubscribeVmss.url == v2ray_url).first()) if data is not None: if (data.death_count is None or data.death_count < global_variable.get_conf_int( "BASE_DEATH_COUNT", default=10)): logger.debug("vmess crawl again {}".format(v2ray_url)) new_db = global_variable.get_db() new_db.query(SubscribeVmss).filter( SubscribeVmss.id == data.id).update({ SubscribeVmss.death_count: int( global_variable.get_conf_int("BASE_DEATH_COUNT", default=30) / 2), }) new_db.commit() return True if v2ray_url.startswith("vmess://"): # vmess try: logger.debug("new vmess is {}".format(v2ray_url)) v = json.loads( utils.base64_decode(v2ray_url.replace("vmess://", ""))) new_db = global_variable.get_db() new_db.add( SubscribeVmss( url=v2ray_url, network_protocol_type="" if v.get("net") is None else v.get("net"), death_count=global_variable.get_conf_int( "BASE_DEATH_COUNT", default=30), next_at=0, is_closed=False, interval=int(interval), crawl_id=int(crawl_id), conf_details=v, )) new_db.commit() return True except sqlalchemy.exc.IntegrityError: pass except (UnicodeDecodeError, json.decoder.JSONDecodeError): pass except: logger.error("err: {}".format(traceback.format_exc())) return False except: logger.error("err: {}".format(traceback.format_exc())) return True
def crawl_by_subscribe(): data_list = (global_variable.get_db().query(SubscribeCrawl).filter( SubscribeCrawl.next_at <= utils.now()).filter( SubscribeCrawl.is_closed == False).all()) for data in data_list: try: get_data_from_network(data) new_db = global_variable.get_db() new_db.query(SubscribeCrawl).filter( SubscribeCrawl.id == data.id).update({ SubscribeCrawl.next_at: int( random.uniform(0.5, 1.5) * data.interval + utils.now()) }) new_db.commit() except: traceback.print_exc() logger.debug("已经完成了\t{}\t个订阅节点的更新".format(len(data_list)))
def get_new_db(): return (global_variable.get_db().query(SubscribeVmss).filter( SubscribeVmss.death_count >= 0).filter( or_(SubscribeVmss.is_closed == False, SubscribeVmss.is_closed is None) ).filter(SubscribeVmss.speed_youtube > 0).filter( SubscribeVmss.network_delay_youtube > 0).filter( SubscribeVmss.speed_internet > 0).filter( SubscribeVmss.network_delay_internet > 0).filter( SubscribeVmss.speed_google > 0).filter( SubscribeVmss.network_delay_google > 0).filter( SubscribeVmss.next_at > 0))
def check_link_alive_by_youtube(data: SubscribeVmss): try: speed, network_delay = check_by_v2ray_url("https://www.youtube.com/") new_db = global_variable.get_db() new_db.query(SubscribeVmss).filter(SubscribeVmss.id == data.id).update( { SubscribeVmss.speed_youtube: speed, SubscribeVmss.network_delay_youtube: network_delay, } ) new_db.commit() return True if speed >= 0 else False except: logger.error(traceback.format_exc()) return False
def check_link_alive_by_internet(data: SubscribeVmss): try: speed, network_delay = check_by_v2ray_url( "http://cachefly.cachefly.net/1mb.test" ) new_db = global_variable.get_db() new_db.query(SubscribeVmss).filter(SubscribeVmss.id == data.id).update( { SubscribeVmss.speed_internet: speed, SubscribeVmss.network_delay_internet: network_delay, } ) new_db.commit() return True if speed >= 0 else False except: logger.error(traceback.format_exc()) return False
def check_link_alive(): logger.info("starting check vpn node......") while True: try: data_list = ( global_variable.get_db() .query(SubscribeVmss) .filter( or_( SubscribeVmss.death_count > 0, SubscribeVmss.death_count == None, ) ) .filter( or_( SubscribeVmss.next_at < utils.now(), SubscribeVmss.next_at == None, ) ) .order_by(SubscribeVmss.next_at) .all() ) if len(data_list) <= 0: logger.debug("暂时没有待检测节点") time.sleep(20) continue else: for i, data in enumerate(data_list): try: update_v2ray_conf(data.url) death_count = data.death_count alive = ( check_link_alive_by_google(data) + check_link_alive_by_youtube(data) + check_link_alive_by_internet(data) ) if alive <= 0: death_count -= 3 - alive else: if death_count < 0: death_count = 0 death_count += alive new_db = global_variable.get_db() new_db.query(SubscribeVmss).filter( SubscribeVmss.id == data.id ).update( { SubscribeVmss.next_at: int( random.uniform(0.5, 1.5) * data.interval ) + int(time.time()), SubscribeVmss.death_count: death_count, } ) new_db.commit() except: logger.error(traceback.format_exc()) finally: time.sleep(5) # logger.info("第{}个节点监测完成".format(i+1)) logger.debug("{}个节点检测完成".format(i + 1)) except: logger.error(traceback.format_exc()) time.sleep(10) finally: time.sleep(10)