def _analyze_ssr(self, nodes: list) -> list:
proxy_list = []
for node in nodes:
decode_proxy = node.decode('utf-8')[6:]
if not decode_proxy or decode_proxy.isspace():
self._debug_printer('ssr节点信息为空，跳过该节点')
continue
proxy_str = self.url_decode(decode_proxy).decode('utf-8')
parts = proxy_str.split(':')
if len(parts) != 6:
logger.error('该ssr节点解析失败，链接:{}'.format(node))
continue
info = {
'server': parts[0],
'port': parts[1],
'protocol': parts[2],
'method': parts[3],
'obfs': parts[4]
}
password_params = parts[5].split('/?')
info['password'] = self.url_decode(password_params[0]).decode('utf-8')
params = password_params[1].split('&')
for p in params:
key_value = p.split('=')
info[key_value[0]] = self.url_decode(key_value[1]).decode('utf-8')
proxy_list.append(info)
return proxy_list
def run(self, api=None):
logger.debug(
f">> RUN <{self.action_name}> --> beat_sync[{self.beat_sync}] feature[General]"
)
# 获取任务设置
api = self.set_spider_option() if api is None else api
# 执行核心业务逻辑
try:
# 设置弹性计时器，当目标站点未能在规定时间内渲染到预期范围时自主销毁实体
# 防止云彩姬误陷“战局”被站长过肩摔
self.get_html_handle(api=api,
url=self.register_url,
wait_seconds=45)
# 注册账号
self.sign_up(api)
# 进入站点并等待核心元素渲染完成
self.wait(api, 40, "//div[@class='card-body']")
# 根据原子类型订阅的优先顺序 依次捕获
self.capture_subscribe(api)
except TimeoutException:
logger.error(
f'>>> TimeoutException <{self.action_name}> -- {self.register_url}'
)
except WebDriverException as e:
logger.error(f">>> WebDriverException <{self.action_name}> -- {e}")
except (HTTPError, ConnectionRefusedError, ConnectionResetError):
pass
except Exception as e:
logger.warning(f">>> Exception <{self.action_name}> -- {e}")
finally:
api.quit()
def sever_chan(title: str = None, message: str = None) -> bool:
"""
调用SERVER酱微信提示
@param title: 标题最大256
@param message: 正文，支持markdown，最大64kb
@return:
"""
if not isinstance(title, str) or not isinstance(message, str):
return False
import requests
url = f"http://sc.ftqq.com/{SERVER_CHAN_SCKEY}.send"
params = {'text': title, 'desp': message}
try:
res = requests.get(url, params=params)
res.raise_for_status()
if res.status_code == 200 and res.json().get("errmsg") == 'success':
logger.success("Server酱设备通知已发送~")
return True
except requests.exceptions.HTTPError:
err_ = "Server酱404！！！可能原因为您的SCKEY未填写或已重置，请访问 http://sc.ftqq.com/3.version 查看解决方案\n" \
"工作流将保存此漏洞数据至error.log 并继续运行，希望您常来看看……"
logger.error(err_)
send_email(err_, to_='self')
def pop_subs_to_admin(class_: str):
"""
@param class_:
@return:
"""
logger.debug("<SuperAdmin> -- 获取订阅")
from src.BusinessLogicLayer.cluster.sailor import manage_task
try:
# 获取该类型订阅剩余链接
remain_subs: list = RedisClient().sync_remain_subs(
REDIS_SECRET_KEY.format(class_))
while True:
# 若无可用链接则返回错误信息
if remain_subs.__len__() == 0:
logger.error(f'<SuperAdmin> -- 无可用<{class_}>订阅')
return {'msg': 'failed', 'info': f"无可用<{class_}>订阅"}
else:
# 从池中获取(最新加入的)订阅s-e
subs, end_life = remain_subs.pop()
# 将s-e加入缓冲队列，该队列将被ddt的refresh工作流同过期链接一同删除
# 使用缓冲队列的方案保证节拍同步，防止过热操作/失误操作贯穿Redis
# 既当管理员通过此接口获取链接时，被返回的链接不会直接从池中删去
# 而是触发缓冲机制，既将该链接标记后加入apollo缓冲队列
# apollo队列内的元素都是欲删除缓存，当ddt发动后会一次性情况当前所有的缓存
# 对订阅进行质量粗检
# if subs2node(subs=subs, cache_path=False, timeout=2)['node'].__len__() <= 3:
# logger.debug(f"<check> BadLink -- {subs}")
# continue
# 使用节拍同步线程锁发起连接池回滚指令,仅生成/同步一枚原子任务
threading.Thread(target=manage_task,
kwargs={
"class_": class_,
"only_sync": True
}).start()
logger.success('管理员模式--链接分发成功')
# 立即执行链接解耦，将同一账号的所有订阅移除
# beat_sync =True立即刷新，False延迟刷新（节拍同步）
threading.Thread(target=detach,
kwargs={
"subscribe": subs,
'beat_sync': True
}).start()
return {
'msg': 'success',
'subscribe': subs,
'subsType': class_
}
except Exception as e:
logger.exception(e)
return {'msg': 'failed', 'info': str(e)}
def sign_in(self, api: Chrome, email=None, password=None):
"""登录行为"""
email = self.email if email is None else email
password = self.password if password is None else password
self.wait(api, timeout=5, tag_xpath_str="//button")
api.find_element_by_id("email").send_keys(email)
api.find_element_by_id("password").send_keys(password)
api.find_element_by_tag_name("button").click()
try:
api.find_element_by_xpath("//h2[@id='swal2-title']")
logger.error("FAILED Incorrect account or password.")
except NoSuchElementException:
pass
def send_email(msg, to_: List[str] or str or set, headers: str = None):
"""
发送运维信息，该函数仅用于发送简单文本信息
:param msg: 正文内容
:param to_: 发送对象
1. str
to_ == 'self'，发送给“自己”
2. List[str]
传入邮箱列表，群发邮件（内容相同）。
:param headers:
:@todo 加入日志读取功能（open file）以及富文本信息功能（html邮件）
:return: 默认为'<V2Ray云彩姬>运维日志'
"""
headers = headers if headers else '<V2Ray云彩姬>运维日志'
sender = SMTP_ACCOUNT.get('email')
password = SMTP_ACCOUNT.get('sid')
smtp_server = 'smtp.qq.com'
message = MIMEText(msg, 'plain', 'utf-8')
message['From'] = Header('ARAI.DM', 'utf-8') # 发送者
message['Subject'] = Header(f"{headers}", 'utf-8')
server = smtplib.SMTP_SSL(smtp_server, 465)
# 输入转换
if to_ == 'self':
to_ = set(sender, )
if isinstance(to_, str):
to_ = [
to_,
]
if isinstance(to_, list):
to_ = set(to_)
if not isinstance(to_, set):
return False
try:
server.login(sender, password)
for to in to_:
try:
message['To'] = Header(to, 'utf-8') # 接收者
server.sendmail(sender, to, message.as_string())
logger.success("发送成功->{}".format(to))
except smtplib.SMTPRecipientsRefused:
logger.warning('邮箱填写错误或不存在->{}'.format(to))
except Exception as e:
logger.error('>>> 发送失败 || {}'.format(e))
finally:
server.quit()
def run(self):
logger.info("DO -- <{}>:beat_sync:{}".format(self.action_name, self.beat_sync))
# ======================================
# 获取任务设置
# ======================================
api = self.set_spider_option()
# ======================================
# 执行核心业务逻辑
# ======================================
try:
# 跳转网页
# 设置超时（堡垒机/Cloudflare/WebError/流量劫持/IP防火墙）引发TimeoutException
self.get_html_handle(api=api, url=self.register_url, wait_seconds=15)
# 注册账号
self.sign_up(api)
# 等待核心元素加载/渲染
self.wait(api, 20, "//div[@class='card-body']")
# 捕获各类型订阅
if self.hyper_params['v2ray']:
self.load_any_subscribe(
api,
"//div[@class='buttons']//a[contains(@class,'v2ray')]",
'data-clipboard-text',
'v2ray'
)
elif self.hyper_params['ssr']:
self.load_any_subscribe(
api,
"""//a[@onclick="importSublink('ssr')"]/..//a[contains(@class,'copy')]""",
'data-clipboard-text',
'ssr'
)
# elif self.hyper_params['trojan']: ...
# elif self.hyper_params['kit']: ...
# elif self.hyper_params['qtl']: ...
except TimeoutException:
logger.error(f'>>> TimeoutException <{self.action_name}> -- {self.register_url}')
except WebDriverException as e:
logger.error(f">>> WebDriverException <{self.action_name}> -- {e}")
except Exception as e:
logger.exception(f">>> Exception <{self.action_name}> -- {e}")
finally:
api.quit()
def collector(self,
silence: bool = True,
debug: bool = False,
page_num: int = 26,
sleep_node: int = 5):
"""
STAFF site collector
Use Selenium to obtain small batch samples through Google Search Engine
(according to statistics, there are about 245 legal sites worldwide)
The collection principle is roughly as follows:
Use the characteristic word SEO to detect whether the target site exists `/staff` page content.
:param silence: True无头启动（默认），False显示启动（请仅在调试时启动）
:param debug:
:param page_num: 采集“页数”，一页约10条检索结果，依上文所述，此处page_num设置为26左右
:param sleep_node: 每采集多少页进行一次随机时长的休眠，默认sleep_node为5
:return:
"""
logger.info(
f"Successfully obtained interface permissions -> {StaffCollector.__name__}"
)
try:
# 采集器实例化
StaffCollector(
silence=silence,
# cache_path 为采集到的站点链接输出目录
cache_path=self._cache_path_staff_hosts,
chromedriver_path=CHROMEDRIVER_PATH,
debug=debug).run(page_num=page_num, sleep_node=sleep_node)
except CollectorSwitchError:
logger.error(
"<StaffCollector> Traffic interception is detected, and the system is taking a backup plan"
)
except IndexError:
logger.warning(
"<StaffCollector> An error occurred while switching the page number"
)
except NoSuchWindowException:
logger.error("<StaffCollector> The Chromedriver exited abnormally")
except Exception as e:
logger.exception(f"<StaffCollector> {e}")
def run(self):
logger.info("DO -- <{}>:beat_sync:{}".format(self.__class__.__name__,
self.beat_sync))
api = self.set_spider_option()
api.get(self.register_url)
try:
self.sign_up(api)
self.wait(api, 20, "//div[@class='card-body']")
# get v2ray link
if self.hyper_params['v2ray']:
self.load_any_subscribe(
api,
"//div[@class='buttons']//a[contains(@class,'v2ray')]",
'data-clipboard-text', 'v2ray')
# get ssr link
elif self.hyper_params['ssr']:
self.load_any_subscribe(
api,
"""//a[@onclick="importSublink('ssr')"]/..//a[contains(@class,'copy')]""",
'data-clipboard-text', 'ssr')
# if self.hyper_params['trojan']: ...
# if self.hyper_params['kit']: ...
# if self.hyper_params['qtl']: ...
except TimeoutException:
logger.error(
f'>>> TimeoutException <{self.__class__.__name__}> -- {self.register_url}'
)
# except WebDriverException as e:
# logger.exception(f">>> Exception <{self.__class__.__name__}> -- {e}")
except Exception as e:
logger.exception(
f">>> Exception <{self.__class__.__name__}> -- {e}")
finally:
# Middleware.hera.put_nowait("push")
api.quit()
def check_config(call_driver: bool = False):
chromedriver_not_found_error = "<ScaffoldGuider> ForceRun || ChromedriverNotFound ||" \
"未查找到chromedriver驱动，请根据技术文档正确配置\n" \
">>> https://github.com/QIN2DIM/V2RayCloudSpider"
# if not all(SMTP_ACCOUNT.values()):
# logger.warning('您未正确配置<通信邮箱>信息(SMTP_ACCOUNT)')
# if not SERVERCHAN_SCKEY:
# logger.warning("您未正确配置<Server酱>的SCKEY")
if not all(
[REDIS_SLAVER_DDT.get("host"),
REDIS_SLAVER_DDT.get("password")]):
logger.warning('您未正确配置<Redis-Slave> 本项目资源拷贝功能无法使用，但不影响系统正常运行。')
if not all([REDIS_MASTER.get("host"), REDIS_MASTER.get("password")]):
logger.error("您未正确配置<Redis-Master> 此配置为“云彩姬”的核心组件，请配置后重启项目！")
sys.exit()
# 当需要调用的接口涉及到driver操作时抛出
if call_driver and not os.path.exists(CHROMEDRIVER_PATH):
logger.error(chromedriver_not_found_error)
sys.exit()
def run(self, class_: str) -> None:
"""
Data disaster tolerance or one class_
@param class_: subscribe type `ssr` or `v2ray` or `trojan` ...
@return:
"""
key_name = REDIS_SECRET_KEY.format(class_)
self.refresh(key_name, cross_threshold=6)
# 数据拷贝 ... -> self
for subscribe, end_life in self.db.hgetall(key_name).items():
self.docker.update({subscribe: end_life})
# logger.info("{} {}".format(key_name, subscribe))
# 映射迁移 acm <- ...
try:
self.acm.get_driver().hset(key_name, mapping=self.docker)
except redis.exceptions.DataError:
logger.warning(f'({class_}):缓存可能被击穿或缓存为空，请系统管理员及时维护链接池！')
except redis.exceptions.ConnectionError:
logger.error(f"redis-slave {self.redis_virtual} 或宕机")
def run(self):
logger.info("DO -- <{}>:beat_sync:{}".format(self.action_name, self.beat_sync))
# 获取任务设置
api = self.set_spider_option()
# 执行核心业务逻辑
try:
# 设置弹性计时器，当目标站点未能在规定时间内渲染到预期范围时自主销毁实体
# 防止云彩姬误陷“战局”被站长过肩摔
self.get_html_handle(api=api, url=self.register_url, wait_seconds=45)
# 注册账号
self.sign_up(api)
# 进入站点并等待核心元素渲染完成
self.wait(api, 40, "//div[@class='card-body']")
# 根据原子类型订阅的优先顺序 依次捕获
if self.hyper_params['v2ray']:
self.load_any_subscribe(
api,
"//div[@class='buttons']//a[contains(@class,'v2ray')]",
'data-clipboard-text',
'v2ray'
)
elif self.hyper_params['ssr']:
self.load_any_subscribe(
api,
"""//a[@onclick="importSublink('ssr')"]/..//a[contains(@class,'copy')]""",
'data-clipboard-text',
'ssr'
)
# elif self.hyper_params['trojan']: ...
# elif self.hyper_params['kit']: ...
# elif self.hyper_params['qtl']: ...
except TimeoutException:
logger.error(f'>>> TimeoutException <{self.action_name}> -- {self.register_url}')
except WebDriverException as e:
logger.error(f">>> WebDriverException <{self.action_name}> -- {e}")
except Exception as e:
logger.exception(f">>> Exception <{self.action_name}> -- {e}")
finally:
api.quit()
def set_task2url_cache(task_name, register_url, subs):
"""
@param task_name: XXCloud
@param register_url:
@param subs:
@return:
"""
from src.config import SINGLE_DEPLOYMENT
from src.BusinessCentralLayer.middleware.work_io import Middleware
try:
docker = {
urlparse(register_url).netloc: {
"name": task_name,
"type": urlparse(subs).netloc
}
}
Middleware.theseus.update(docker)
# TODO 功能复杂化
if not SINGLE_DEPLOYMENT:
pass
except Exception as e:
logger.error(f"<CharMap> {task_name} || {e}")
def parse(self, **kwargs):
"""
:return:
"""
api: Chrome = kwargs.get("api")
self.obj_parser.update({"parse_url": self.register_url})
# ----------------------------------------
# 解析可用流量和可用时长
# 优先调用，等待流体动画加载完成[耗时任务]
# 以保证后续解析无需等待
# ----------------------------------------
fluid = set()
fluid_density = []
i = 0
while True:
try:
i += 1
card_body = api.find_elements_by_xpath(
"//div[@class='card-body']")[:2]
card_body = [tag.text.strip() for tag in card_body]
fluid.update(card_body)
fluid_density.append(len(fluid))
# 流体释放
if len(fluid_density) < 10 or len(fluid) < 3:
continue
# 流体相对均衡
if max(fluid_density[:10]) == min(fluid_density[:10]):
self.obj_parser.update({
"time": card_body[0],
"flow": card_body[-1]
})
break
except StaleElementReferenceException:
pass
# 存储cookie
with open("123.json", "w", encoding="utf8") as f:
f.write(json.dumps(api.get_cookies()))
# 读取cookie
# cookie_json = " ".join([f"{i['name']}={i['value']};" for i in json.loads(f.read())])
# ----------------------------------------
# 解析站点名称
# ----------------------------------------
try:
parse_name = api.find_element_by_xpath(
"//aside//div[@class='sidebar-brand']").text.strip()
self.obj_parser.update({"parse_name": parse_name})
except WebDriverException:
logger.error(
f"<SSPanelParserError> Site name resolution failed -- {self.register_url}"
)
# ----------------------------------------
# 解析站点公告
# ----------------------------------------
reference_links = {}
try:
card_body = api.find_elements_by_xpath(
"//div[@class='card-body']")[4]
self.obj_parser.update({"desc": card_body.text.strip()})
related_href = card_body.find_elements_by_tag_name("a")
for tag in related_href:
href = tag.get_attribute("href")
if href:
href = href.strip()
if "https" not in href:
href = f"{self.register_url}{href}"
href_desc = tag.text.strip() if tag.text else href
reference_links.update({href: href_desc})
self.obj_parser.update({"reference_links": reference_links})
except WebDriverException:
logger.error(
f"<SSPanelParserError> Site announcement parsing error -- {self.register_url}"
)
# ----------------------------------------
# 解析[链接导入]
# ----------------------------------------
subscribes = {}
support = []
try:
# 清洗订阅链接
soup = BeautifulSoup(api.page_source, 'html.parser')
for i in soup.find_all("a"):
if i.get("data-clipboard-text"):
subscribes.update(
{i.get("data-clipboard-text"): i.text.strip()})
# 识别支持的订阅类型
buttons = api.find_elements_by_xpath("//div[@class='card'][2]//a")
for tag in buttons:
support_ = tag.get_attribute("class")
if support_:
support_ = [
i for i in
[i for i in support_.split() if i.startswith("btn-")]
if i not in [
'btn-icon', 'btn-primary', 'btn-lg', 'btn-round',
'btn-progress'
]
]
if len(support_) == 1:
class_name = support_[0].replace("btn-", "")
support.append(class_name)
# 残差补全
for tag in subscribes.values():
if "surge" in tag.lower():
support.append("surge")
if "ssr" in tag.lower():
support.append("ssr")
self.obj_parser.update({
"subscribes": subscribes,
"support": list(set(support))
})
except WebDriverException:
logger.error(
f"<SSPanelParserError> Site subscription resolution failed -- {self.register_url}"
)
self.obj_parser.update({
"email": self.email,
"password": self.password,
"recently_login": datetime.now(tz=TIME_ZONE_CN)
})
return self.obj_parser
