def __init__(self):
super(RedisDataDisasterTolerance, self).__init__()
from src.BusinessCentralLayer.setting import REDIS_SLAVER_DDT
if not REDIS_SLAVER_DDT.get('host'):
logger.warning('未设置数据容灾服务器，该职能将由Master执行')
# 拷贝参数
redis_virtual = REDIS_MASTER
# 改动浅拷贝数据库
redis_virtual.update({'db': redis_virtual['db'] + 1})
logger.debug("备份重定向 --> {}".format(redis_virtual))
else:
redis_virtual = REDIS_SLAVER_DDT
# 容器初始化
self.docker = {}
try:
self.acm = RedisClient(host=redis_virtual['host'],
port=redis_virtual['port'],
password=redis_virtual['password'])
logger.info("DDT: Master({}) -> Slaver({})".format(
REDIS_MASTER['host'], redis_virtual['host']))
except redis.exceptions.ConnectionError as e:
logger.exception(e)
finally:
self.redis_virtual = redis_virtual
def startup(self):
# --------------------------------------------------
# 开辟进程
# --------------------------------------------------
try:
# 部署flask
process_server = multiprocessing.Process(target=self.run_server,
name="VirtualStation")
# 执行业务
process_business = multiprocessing.Process(
target=self.run_business, name="Adapter")
# 执行多进程任务
process_server.start()
process_business.start()
# 简单监测 主动回收进程
while True:
if not process_business.is_alive():
process_server.terminate()
return True
except TypeError or AttributeError as e:
logger.exception(e)
finally:
logger.success('<ScaffoldGuider> End the V2RayCloudSpider')
print(">>> 程序執行結束 請手動關閉交互窗口")
def _deploy_jobs(self):
"""
@return:
"""
try:
for docker in self.dockers:
# 添加任务
self.scheduler_.add_job(
func=docker['api'],
trigger=IntervalTrigger(
seconds=self.interval_[docker['name']]),
id=docker['name'],
jitter=5)
# 打印日志
logger.info(
f'<BlockingScheduler> Add job -- <{docker["name"]}>'
f' IntervalTrigger: {self.interval_[docker["name"]]}s')
# 启动任务
self.scheduler_.start()
except KeyboardInterrupt as err:
logger.stop('Forced stop ||{}'.format(err))
except Exception as err:
logger.exception(f'<BlockingScheduler>||{err}')
def deploy_jobs(self):
try:
for docker in self.dockers:
# 添加任务
job = self.scheduler_.add_job(
func=docker['api'],
trigger=IntervalTrigger(
seconds=self.interval_[docker['name']]),
id=docker['name'],
# 定時抖動
jitter=5,
# 定任务设置最大实例并行数
max_instances=16,
# 把多个排队执行的同一个哑弹任务变成一个
coalesce=True,
)
self.jobs.append(job)
# 打印日志
logger.info(
f'<BlockingScheduler> Add job -- <{docker["name"]}>'
f' IntervalTrigger: {self.interval_[docker["name"]]}s')
# 启动任务
self.scheduler_.start()
except KeyboardInterrupt:
self.scheduler_.shutdown(wait=False)
logger.warning(
"<BlockingScheduler> The admin forcibly terminated the scheduled task"
)
except Exception as err:
logger.exception(f'<BlockingScheduler>||{err}')
def startup() -> None:
process_list = []
try:
# 部署<单进程多线程>定时任务
if ENABLE_DEPLOY['global']:
process_list.append(
multiprocessing.Process(target=_SystemEngine.run_deploy,
name='deploymentTimingTask'))
# 部署flask
if ENABLE_SERVER:
process_list.append(
multiprocessing.Process(target=_SystemEngine.run_server,
name='deploymentFlaskAPI'))
# 执行多进程任务
for process_ in process_list:
logger.success(f'<SystemProcess> Startup -- {process_.name}')
process_.start()
# 添加阻塞
for process_ in process_list:
process_.join()
except TypeError or AttributeError as e:
logger.exception(e)
send_email(f"[程序异常终止]{str(e)}", to_='self')
except KeyboardInterrupt:
# FIXME 确保进程间不产生通信的情况下终止
logger.debug('<SystemProcess> Received keyboard interrupt signal')
for process_ in process_list:
process_.terminate()
finally:
logger.success('<SystemProcess> End the V2RayCloudSpider')
def pop_subs_to_admin(class_: str):
"""
@param class_:
@return:
"""
logger.debug("<SuperAdmin> -- 获取订阅")
from src.BusinessLogicLayer.cluster.sailor import manage_task
try:
# 获取该类型订阅剩余链接
remain_subs: list = RedisClient().sync_remain_subs(
REDIS_SECRET_KEY.format(class_))
while True:
# 若无可用链接则返回错误信息
if remain_subs.__len__() == 0:
logger.error(f'<SuperAdmin> -- 无可用<{class_}>订阅')
return {'msg': 'failed', 'info': f"无可用<{class_}>订阅"}
else:
# 从池中获取(最新加入的)订阅s-e
subs, end_life = remain_subs.pop()
# 将s-e加入缓冲队列，该队列将被ddt的refresh工作流同过期链接一同删除
# 使用缓冲队列的方案保证节拍同步，防止过热操作/失误操作贯穿Redis
# 既当管理员通过此接口获取链接时，被返回的链接不会直接从池中删去
# 而是触发缓冲机制，既将该链接标记后加入apollo缓冲队列
# apollo队列内的元素都是欲删除缓存，当ddt发动后会一次性情况当前所有的缓存
# 对订阅进行质量粗检
# if subs2node(subs=subs, cache_path=False, timeout=2)['node'].__len__() <= 3:
# logger.debug(f"<check> BadLink -- {subs}")
# continue
# 使用节拍同步线程锁发起连接池回滚指令,仅生成/同步一枚原子任务
threading.Thread(target=manage_task,
kwargs={
"class_": class_,
"only_sync": True
}).start()
logger.success('管理员模式--链接分发成功')
# 立即执行链接解耦，将同一账号的所有订阅移除
# beat_sync =True立即刷新，False延迟刷新（节拍同步）
threading.Thread(target=detach,
kwargs={
"subscribe": subs,
'beat_sync': True
}).start()
return {
'msg': 'success',
'subscribe': subs,
'subsType': class_
}
except Exception as e:
logger.exception(e)
return {'msg': 'failed', 'info': str(e)}
def to_sqlite3(docker: dict):
"""
@param docker: {uuid1:{key1:value1, key2:value2, ...}, uuid2:{key1:value1, key2:value2, ...}, ...} len >= 1
@return:
"""
try:
if docker.keys().__len__() >= 1:
docker = [tuple(data.values()) for data in docker.values()]
# logger.success(f'>> STORING -> Sqlite3')
except Exception as e:
logger.exception(e)
finally:
FlowTransferStation(docker=docker).add()
def run(self):
logger.info("DO -- <{}>:beat_sync:{}".format(self.action_name, self.beat_sync))
# ======================================
# 获取任务设置
# ======================================
api = self.set_spider_option()
# ======================================
# 执行核心业务逻辑
# ======================================
try:
# 跳转网页
# 设置超时（堡垒机/Cloudflare/WebError/流量劫持/IP防火墙）引发TimeoutException
self.get_html_handle(api=api, url=self.register_url, wait_seconds=15)
# 注册账号
self.sign_up(api)
# 等待核心元素加载/渲染
self.wait(api, 20, "//div[@class='card-body']")
# 捕获各类型订阅
if self.hyper_params['v2ray']:
self.load_any_subscribe(
api,
"//div[@class='buttons']//a[contains(@class,'v2ray')]",
'data-clipboard-text',
'v2ray'
)
elif self.hyper_params['ssr']:
self.load_any_subscribe(
api,
"""//a[@onclick="importSublink('ssr')"]/..//a[contains(@class,'copy')]""",
'data-clipboard-text',
'ssr'
)
# elif self.hyper_params['trojan']: ...
# elif self.hyper_params['kit']: ...
# elif self.hyper_params['qtl']: ...
except TimeoutException:
logger.error(f'>>> TimeoutException <{self.action_name}> -- {self.register_url}')
except WebDriverException as e:
logger.error(f">>> WebDriverException <{self.action_name}> -- {e}")
except Exception as e:
logger.exception(f">>> Exception <{self.action_name}> -- {e}")
finally:
api.quit()
def _generate_model(self, data: dict, model: dict) -> dict:
self._debug_printer("正在生成配置文件...")
try:
# 参数清洗
if model.get('proxies') is None:
model['proxies'] = data.get('proxy_list')
else:
model['proxies'].extend(data.get('proxy_list'))
# 更新模具
for group in model.get('proxy-groups'):
if group.get('proxies') is None:
group['proxies'] = data.get('proxy_names')
else:
group['proxies'].extend(data.get('proxy_names'))
return model
except AttributeError as e:
logger.exception(e)
def _update_entropy(rc=None):
# 组合entropy标注数据
try:
atomic_queue = []
for i in __entropy__:
work_filed = [
f"{j[0].upper()}" for j in i['hyper_params'].items() if j[-1]
]
work_filed = "&".join(work_filed).strip()
atomic_item = f"|{work_filed}| {i['name']}"
atomic_queue.append(atomic_item)
# 更新列表
if rc is None:
rc = RedisClient()
rc.get_driver().set(name=REDIS_SECRET_KEY.format("__entropy__"),
value="$".join(atomic_queue))
except Exception as e:
logger.exception(e)
def set_up_file_tree(self, root):
"""
--/qinse/V2RaycSpider{verNum}
--BCL
--BLL
--BVL
--Database
--client_depot
--vcs.csv
--logs
--*error.log
--*runtime.log
--temp_cache
--*AnyTempCacheFile...
--*CrawlFetchHistory.txt
--fake_useragent_0.1.11.json
--*tests
"""
# 检查默认下载地址是否残缺 深度优先初始化系统文件
for child_ in root:
if not os.path.exists(child_):
self.flag = True
try:
# 初始化文件夹
if os.path.isdir(
child_) or not os.path.splitext(child_)[-1]:
os.mkdir(child_)
logger.success(f"系统文件链接成功->{child_}")
# 初始化文件
else:
if child_ == SERVER_PATH_DEPOT_VCS:
try:
with open(child_,
'w',
encoding='utf-8',
newline='') as fpx:
csv.writer(fpx).writerow(
['version', 'title'])
logger.success(f"系统文件链接成功->{child_}")
except Exception as ep:
logger.exception(f"Exception{child_}{ep}")
except Exception as ep:
logger.exception(ep)
def collector(self,
silence: bool = True,
debug: bool = False,
page_num: int = 26,
sleep_node: int = 5):
"""
STAFF site collector
Use Selenium to obtain small batch samples through Google Search Engine
(according to statistics, there are about 245 legal sites worldwide)
The collection principle is roughly as follows:
Use the characteristic word SEO to detect whether the target site exists `/staff` page content.
:param silence: True无头启动（默认），False显示启动（请仅在调试时启动）
:param debug:
:param page_num: 采集“页数”，一页约10条检索结果，依上文所述，此处page_num设置为26左右
:param sleep_node: 每采集多少页进行一次随机时长的休眠，默认sleep_node为5
:return:
"""
logger.info(
f"Successfully obtained interface permissions -> {StaffCollector.__name__}"
)
try:
# 采集器实例化
StaffCollector(
silence=silence,
# cache_path 为采集到的站点链接输出目录
cache_path=self._cache_path_staff_hosts,
chromedriver_path=CHROMEDRIVER_PATH,
debug=debug).run(page_num=page_num, sleep_node=sleep_node)
except CollectorSwitchError:
logger.error(
"<StaffCollector> Traffic interception is detected, and the system is taking a backup plan"
)
except IndexError:
logger.warning(
"<StaffCollector> An error occurred while switching the page number"
)
except NoSuchWindowException:
logger.error("<StaffCollector> The Chromedriver exited abnormally")
except Exception as e:
logger.exception(f"<StaffCollector> {e}")
def run(self):
logger.info("DO -- <{}>:beat_sync:{}".format(self.__class__.__name__,
self.beat_sync))
api = self.set_spider_option()
api.get(self.register_url)
try:
self.sign_up(api)
self.wait(api, 20, "//div[@class='card-body']")
# get v2ray link
if self.hyper_params['v2ray']:
self.load_any_subscribe(
api,
"//div[@class='buttons']//a[contains(@class,'v2ray')]",
'data-clipboard-text', 'v2ray')
# get ssr link
elif self.hyper_params['ssr']:
self.load_any_subscribe(
api,
"""//a[@onclick="importSublink('ssr')"]/..//a[contains(@class,'copy')]""",
'data-clipboard-text', 'ssr')
# if self.hyper_params['trojan']: ...
# if self.hyper_params['kit']: ...
# if self.hyper_params['qtl']: ...
except TimeoutException:
logger.error(
f'>>> TimeoutException <{self.__class__.__name__}> -- {self.register_url}'
)
# except WebDriverException as e:
# logger.exception(f">>> Exception <{self.__class__.__name__}> -- {e}")
except Exception as e:
logger.exception(
f">>> Exception <{self.__class__.__name__}> -- {e}")
finally:
# Middleware.hera.put_nowait("push")
api.quit()
def _analyze_rss(self) -> dict:
headers = {'User-Agent': 'Clash For Python'}
proxy_list = {
'proxy_list': [],
'proxy_names': []
}
for run_entity in self._rss_pool:
self._debug_printer(run_entity)
res = requests.get(url=run_entity, headers=headers, timeout=5, proxies=self._PROXIES)
try:
b64code_: bytes = base64.b64decode(res.text)
except Exception as r:
logger.exception(r)
continue
node_list = b64code_.splitlines()
v2ray_urls = []
ss_urls = []
ssr_urls = []
for node in node_list:
if node.startswith(b'vmess://'):
v2ray_urls.append(node)
elif node.startswith(b'ss://'):
ss_urls.append(node)
elif node.startswith(b'ssr://'):
ssr_urls.append(node)
else:
pass
clash_node = {}
if v2ray_urls.__len__() > 0:
decode_proxy: list = self._analyze_v2ray(nodes=v2ray_urls)
clash_node: dict = self._v2ray_to_yaml(arr=decode_proxy)
elif ssr_urls.__len__() > 0:
decode_proxy: list = self._analyze_ssr(nodes=ssr_urls)
clash_node: dict = self._ssr_to_yaml(arr=decode_proxy)
proxy_list['proxy_list'].extend(clash_node['proxy_list'])
proxy_list['proxy_names'].extend(clash_node['proxy_names'])
self._debug_printer(f"共发现:{proxy_list['proxy_names'].__len__()}个节点")
return proxy_list
def run(self):
logger.info("DO -- <{}>:beat_sync:{}".format(self.action_name, self.beat_sync))
# 获取任务设置
api = self.set_spider_option()
# 执行核心业务逻辑
try:
# 设置弹性计时器，当目标站点未能在规定时间内渲染到预期范围时自主销毁实体
# 防止云彩姬误陷“战局”被站长过肩摔
self.get_html_handle(api=api, url=self.register_url, wait_seconds=45)
# 注册账号
self.sign_up(api)
# 进入站点并等待核心元素渲染完成
self.wait(api, 40, "//div[@class='card-body']")
# 根据原子类型订阅的优先顺序 依次捕获
if self.hyper_params['v2ray']:
self.load_any_subscribe(
api,
"//div[@class='buttons']//a[contains(@class,'v2ray')]",
'data-clipboard-text',
'v2ray'
)
elif self.hyper_params['ssr']:
self.load_any_subscribe(
api,
"""//a[@onclick="importSublink('ssr')"]/..//a[contains(@class,'copy')]""",
'data-clipboard-text',
'ssr'
)
# elif self.hyper_params['trojan']: ...
# elif self.hyper_params['kit']: ...
# elif self.hyper_params['qtl']: ...
except TimeoutException:
logger.error(f'>>> TimeoutException <{self.action_name}> -- {self.register_url}')
except WebDriverException as e:
logger.error(f">>> WebDriverException <{self.action_name}> -- {e}")
except Exception as e:
logger.exception(f">>> Exception <{self.action_name}> -- {e}")
finally:
api.quit()
def run(self, class_: str) -> None:
"""
Data disaster tolerance or one class_
@param class_: subscribe type `ssr` or `v2ray` or `trojan` ...
@return:
"""
key_name = REDIS_SECRET_KEY.format(class_)
self.refresh(key_name, cross_threshold=6)
# 数据拷贝 ... -> self
for subscribe, end_life in self.db.hgetall(key_name).items():
self.docker.update({subscribe: end_life})
# logger.info("{} {}".format(key_name, subscribe))
# 映射迁移 acm <- ...
try:
self.acm.get_driver().hset(key_name, mapping=self.docker)
except redis.exceptions.DataError:
logger.warning(f'({class_}):缓存可能被击穿或缓存为空，请系统管理员及时维护链接池！')
except redis.exceptions.ConnectionError:
logger.error(f"redis-slave {self.redis_virtual} 可能宕机")
except Exception as e:
logger.exception(e)
def control_driver(self, shunt_pointer):
try:
shunt_pointer()
except Exception as e:
logger.exception(e)
