def add_new_vmess(
v2ray_url,
crawl_id: int = 0,
interval: int = global_variable.get_conf_int("INTERVAL", default=1800),
) -> bool:
try:
if v2ray_url == "":
return False
# 已经存在了，就不管了
data = (global_variable.get_db().query(SubscribeVmss).filter(
SubscribeVmss.url == v2ray_url).first())
if data is not None:
if (data.death_count is None
or data.death_count < global_variable.get_conf_int(
"BASE_DEATH_COUNT", default=10)):
logger.debug("vmess crawl again {}".format(v2ray_url))
new_db = global_variable.get_db()
new_db.query(SubscribeVmss).filter(
SubscribeVmss.id == data.id).update({
SubscribeVmss.death_count:
int(
global_variable.get_conf_int("BASE_DEATH_COUNT",
default=30) / 2),
})
new_db.commit()
return True
if v2ray_url.startswith("vmess://"): # vmess
try:
logger.debug("new vmess is {}".format(v2ray_url))
v = json.loads(
utils.base64_decode(v2ray_url.replace("vmess://", "")))
new_db = global_variable.get_db()
new_db.add(
SubscribeVmss(
url=v2ray_url,
network_protocol_type=""
if v.get("net") is None else v.get("net"),
death_count=global_variable.get_conf_int(
"BASE_DEATH_COUNT", default=30),
next_at=0,
is_closed=False,
interval=int(interval),
crawl_id=int(crawl_id),
conf_details=v,
))
new_db.commit()
return True
except sqlalchemy.exc.IntegrityError:
pass
except (UnicodeDecodeError, json.decoder.JSONDecodeError):
pass
except:
logger.error("err: {}".format(traceback.format_exc()))
return False
except:
logger.error("err: {}".format(traceback.format_exc()))
return True
def crawl_by_subscribe():
data_list = (global_variable.get_db().query(SubscribeCrawl).filter(
SubscribeCrawl.next_at <= utils.now()).filter(
SubscribeCrawl.is_closed == False).all())
for data in data_list:
try:
get_data_from_network(data)
new_db = global_variable.get_db()
new_db.query(SubscribeCrawl).filter(
SubscribeCrawl.id == data.id).update({
SubscribeCrawl.next_at:
int(
random.uniform(0.5, 1.5) * data.interval + utils.now())
})
new_db.commit()
except:
traceback.print_exc()
logger.debug("已经完成了\t{}\t个订阅节点的更新".format(len(data_list)))
def get_new_db():
return (global_variable.get_db().query(SubscribeVmss).filter(
SubscribeVmss.death_count >= 0).filter(
or_(SubscribeVmss.is_closed == False,
SubscribeVmss.is_closed is None)
).filter(SubscribeVmss.speed_youtube > 0).filter(
SubscribeVmss.network_delay_youtube > 0).filter(
SubscribeVmss.speed_internet > 0).filter(
SubscribeVmss.network_delay_internet > 0).filter(
SubscribeVmss.speed_google > 0).filter(
SubscribeVmss.network_delay_google > 0).filter(
SubscribeVmss.next_at > 0))
def check_link_alive_by_youtube(data: SubscribeVmss):
try:
speed, network_delay = check_by_v2ray_url("https://www.youtube.com/")
new_db = global_variable.get_db()
new_db.query(SubscribeVmss).filter(SubscribeVmss.id == data.id).update(
{
SubscribeVmss.speed_youtube: speed,
SubscribeVmss.network_delay_youtube: network_delay,
}
)
new_db.commit()
return True if speed >= 0 else False
except:
logger.error(traceback.format_exc())
return False
def check_link_alive_by_internet(data: SubscribeVmss):
try:
speed, network_delay = check_by_v2ray_url(
"http://cachefly.cachefly.net/1mb.test"
)
new_db = global_variable.get_db()
new_db.query(SubscribeVmss).filter(SubscribeVmss.id == data.id).update(
{
SubscribeVmss.speed_internet: speed,
SubscribeVmss.network_delay_internet: network_delay,
}
)
new_db.commit()
return True if speed >= 0 else False
except:
logger.error(traceback.format_exc())
return False
def check_link_alive():
logger.info("starting check vpn node......")
while True:
try:
data_list = (
global_variable.get_db()
.query(SubscribeVmss)
.filter(
or_(
SubscribeVmss.death_count > 0,
SubscribeVmss.death_count == None,
)
)
.filter(
or_(
SubscribeVmss.next_at < utils.now(),
SubscribeVmss.next_at == None,
)
)
.order_by(SubscribeVmss.next_at)
.all()
)
if len(data_list) <= 0:
logger.debug("暂时没有待检测节点")
time.sleep(20)
continue
else:
for i, data in enumerate(data_list):
try:
update_v2ray_conf(data.url)
death_count = data.death_count
alive = (
check_link_alive_by_google(data)
+ check_link_alive_by_youtube(data)
+ check_link_alive_by_internet(data)
)
if alive <= 0:
death_count -= 3 - alive
else:
if death_count < 0:
death_count = 0
death_count += alive
new_db = global_variable.get_db()
new_db.query(SubscribeVmss).filter(
SubscribeVmss.id == data.id
).update(
{
SubscribeVmss.next_at: int(
random.uniform(0.5, 1.5) * data.interval
)
+ int(time.time()),
SubscribeVmss.death_count: death_count,
}
)
new_db.commit()
except:
logger.error(traceback.format_exc())
finally:
time.sleep(5)
# logger.info("第{}个节点监测完成".format(i+1))
logger.debug("{}个节点检测完成".format(i + 1))
except:
logger.error(traceback.format_exc())
time.sleep(10)
finally:
time.sleep(10)