def run(self) -> None:
# logger.warning('This is a development server. Do not use it in a production deployment.')
try:
for task_name in self.deploy_cluster:
try:
schedule.every(self.crontab['action']).minutes.do(self.push_task, task_name=task_name)
if ENABLE_DDT:
schedule.every(self.crontab['refresh']).minutes.do(self.rc.refresh,
key_name=REDIS_SECRET_KEY.format(task_name))
logger.success(f"START DDT -- {task_name}")
else:
logger.warning(f'Not Authorized -- DDT({task_name})')
logger.success(f"START TASK -- {task_name}/crontab:{self.crontab['action']} minutes")
except schedule.IntervalError:
logger.error('interval set error')
self.crontab['action'] += 5
while True:
schedule.run_pending()
time.sleep(1)
except Exception as err:
logger.exception('Exception occurred ||{}'.format(err))
noticer.send_email(text_body='{}'.format(err), to='self')
except KeyboardInterrupt as err:
logger.stop('Forced stop ||{}'.format(err))
def push_info(self, user: dict or List[dict]):
if isinstance(user, dict):
user = [
user,
]
elif not isinstance(user, list):
logger.warning('MySQL add_user 调用格式有误')
try:
for user_ in user:
try:
sql = f'INSERT INTO v2raycs (' \
f'domain, subs, class_,end_life,res_time,passable,username,password,email,uuid) VALUES (' \
f'%s, %s, %s,%s, %s, %s,%s, %s, %s,%s)'
val = (user_["domain"], user_["subs"], user_['class_'],
user_['end_life'], user_["res_time"],
user_['passable'], user_['username'],
user_["password"], user_['email'], user_['uuid'])
self.cursor.execute(sql, val)
except KeyError as e:
logger.warning(
f"MySQL数据解析出错，user:dict必须同时包含username、password以及email的键值对{e}"
)
# return 702
except pymysql.err.IntegrityError as e:
logger.warning(
f'{user_["username"]} -- 用户已在库，若需修改用户信息，请使用更新指令{e}')
# return 701
else:
logger.success(f'{user_["username"]} -- 用户添加成功')
# return 700
finally:
self.conn.commit()
self.conn.close()
def sever_chan(title: str = None, message: str = None) -> bool:
"""
调用SERVER酱微信提示
@param title: 标题最大256
@param message: 正文，支持markdown，最大64kb
@return:
"""
if not isinstance(title, str) or not isinstance(message, str):
return False
import requests
url = f"http://sc.ftqq.com/{SERVER_CHAN_SCKEY}.send"
params = {'text': title, 'desp': message}
try:
res = requests.get(url, params=params)
res.raise_for_status()
if res.status_code == 200 and res.json().get("errmsg") == 'success':
logger.success("Server酱设备通知已发送~")
return True
except requests.exceptions.HTTPError:
logger.error(
"Server酱404！！！可能原因为您的SCKEY未填写或已重置，请访问 http://sc.ftqq.com/3.version 查看解决方案"
)
logger.debug('工作流将保存此漏洞数据至error.log 并继续运行，希望您常来看看……')
def push_task(self, task_name: str) -> bool:
"""
@param task_name:
@return:
"""
# 输入参数的数据类型错误
if not isinstance(task_name, str):
logger.error(f'The input type is wrong({task_name})')
return False
# 输入的参数不在模型的权限范围中
if task_name not in self.deploy_cluster:
logger.error(
f'Spelling error in input({task_name}),Please choose from {self.deploy_cluster}'
)
return False
try:
# 判断缓冲队列是否已达单机采集极限
task_name = task_name.lower()
self.rc_len[f'{task_name}'] = self.rc.__len__(
REDIS_SECRET_KEY.format(f'{task_name}'))
logger.info(f'[TEST] ||正在检查({task_name}) 任务队列...')
# 若已达或超过单机采集极限，则休眠任务
if self.rc_len[f"{task_name}"] >= self.cap:
logger.debug(
f'[SLEEP] || 任务队列已满 ({task_name}) ({self.rc_len[f"{task_name}"]}/{self.cap})'
)
return True
finally:
# 无论队列是否已满，执行一次ddt
self.ddt(class_=task_name)
try:
# 执行采集任务，通过self.go决定是否启动协程加速
logger.info(f'[RUN] || ({task_name}) 采集任务启动')
__task__.loads_task(task_name, self.go)
# 判断任务是否完全失败，既单个类型链接的所有采集任务全部失败->Abnormal
if self.rc.__len__(REDIS_SECRET_KEY.format(
f'{task_name}')) < self.rc_len[f'{task_name}']:
logger.error(
f'[CRITICAL]Abnormal collection task({task_name})')
else:
return True
except Exception as e:
# 捕获未知错误
logger.error(
f'[ERROR]{self.__class__.__name__}({task_name}) crawler engine panic {e}'
)
finally:
# 单个类型的链接采集结束
logger.success('[OVER] || 任务结束 {}({})'.format(
self.__class__.__name__, task_name))
async def weather_report(conf):
rep = WeatherReport(conf['url'],{
"tags": {
"region": conf.get('region','')
},
})
logger.success(f'Got "{conf["name"]:18}" report. temp: {rep.temperature}')
return True
def load_any_subscribe(self,
api: Chrome,
element_xpath_str: str,
href_xpath_str: str,
class_: str,
retry=0):
"""
@param api: ChromeDriverObject
@param element_xpath_str: 用于定位链接所在的标签
@param href_xpath_str: 用于取出目标标签的属性值，既subscribe
@param class_: 该subscribe类型，如`ssr`/`v2ray`/`trojan`
@param retry: 失败重试
@todo 使用 retrying 模块替代retry参数实现的功能（引入断网重连，断言重试，行为回滚...）
@return:
"""
self.subscribe = WebDriverWait(api, 30).until(
EC.presence_of_element_located(
(By.XPATH, element_xpath_str))).get_attribute(href_xpath_str)
if self.subscribe:
for x in range(3):
# ['domain', 'subs', 'class_', 'end_life', 'res_time', 'passable','username', 'password', 'email']
try:
domain = urlparse(self.register_url).netloc
res_time = str(datetime.now(TIME_ZONE_CN)).split('.')[0]
passable = 'true'
docker = [
domain, self.subscribe, class_, self.end_life,
res_time, passable, self.username, self.password,
self.email
]
FlexibleDistribute(docker=docker, at_once=self.at_once)
# flexible_distribute(self.subscribe, class_, self.end_life, driver_name=self.__class__.__name__)
logger.success(">> GET <{}> -> {}:{}".format(
self.__class__.__name__, class_, self.subscribe))
break
except Exception as e:
logger.debug(">> FAILED <{}> -> {}:{}".format(
self.__class__.__name__, class_, e))
time.sleep(1)
continue
else:
return None
else:
if retry >= 3:
raise TimeoutException
retry += 1
self.load_any_subscribe(api, element_xpath_str, href_xpath_str,
class_, retry)
async def shutdown(main_task=None):
for task in asyncio.all_tasks():
if task == asyncio.current_task():
continue
try:
# pass
task.cancel()
try:
print("task ex: ",task.exception())
except Exception as ex:
pass
except Exception as ex:
logger.error("Task failed to cancel {}",ex)
finally:
logger.success(f'Finished cleanup, exiting')
def refresh(self, key_name: str) -> None:
"""
原子级链接池刷新，一次性删去所有过期的key_name subscribe
@param key_name:secret_key
@return:
"""
docker: dict = self.db.hgetall(key_name)
# 管理员指令获取的链接
if self.db.hlen(key_name) != 0:
for subscribe, end_life in docker.items():
if self.is_stale(end_life):
logger.debug(f'del-({key_name})--{subscribe}')
self.db.hdel(key_name, subscribe)
logger.success('{}:UPDATE - {}({})'.format(self.__class__.__name__, key_name, self.__len__(key_name)))
else:
logger.warning('{}:EMPTY - {}({})'.format(self.__class__.__name__, key_name, self.__len__(key_name)))
def send_email(msg, to_: List[str] or str or set, headers: str = None):
"""
发送运维信息，该函数仅用于发送简单文本信息
:param msg: 正文内容
:param to_: 发送对象
1. str
to_ == 'self'，发送给“自己”
2. List[str]
传入邮箱列表，群发邮件（内容相同）。
:param headers:
:@todo 加入日志读取功能（open file）以及富文本信息功能（html邮件）
:return: 默认为'<V2Ray云彩姬>运维日志'
"""
headers = headers if headers else '<V2Ray云彩姬>运维日志'
sender = SMTP_ACCOUNT.get('email')
password = SMTP_ACCOUNT.get('sid')
smtp_server = 'smtp.qq.com'
message = MIMEText(msg, 'plain', 'utf-8')
message['From'] = Header('ARAI.DM', 'utf-8') # 发送者
message['Subject'] = Header(f"{headers}", 'utf-8')
server = smtplib.SMTP_SSL(smtp_server, 465)
# 输入转换
if to_ == 'self':
to_ = set(sender, )
if isinstance(to_, str):
to_ = [to_, ]
if isinstance(to_, list):
to_ = set(to_)
if not isinstance(to_, set):
return False
try:
server.login(sender, password)
for to in to_:
try:
message['To'] = Header(to, 'utf-8') # 接收者
server.sendmail(sender, to, message.as_string())
logger.success("发送成功->{}".format(to))
except smtplib.SMTPRecipientsRefused:
logger.warning('邮箱填写错误或不存在->{}'.format(to))
except Exception as e:
logger.error('>>> 发送失败 || {}'.format(e))
finally:
server.quit()
def interface(self, power: int = 8) -> None:
"""
@param power: 协程功率
@return:
"""
# 任务重载
self.offload_task()
# 任务启动
task_list = []
power_ = self.power if self.power else power
for x in range(power_):
task = gevent.spawn(self.launch)
task_list.append(task)
gevent.joinall(task_list)
logger.success(f'<Gevent> mission completed -- <{self.__class__.__name__}>')
def callback(ch, method, properties, body):
# print(1112222223333333)
# print(body.decode())
d = json.loads(body.decode())
# print(66666666)
vid = int(d.pop('id'))
data = {}
data['vid'] = vid
data.update(d)
filter_dict = dict(vid=data.get('vid'),
for_test=int(data.get('for_test')))
video = upsert(Video, data, filter_dict)
video_id = video.id
session.add(video)
session.flush()
table_id = video.id
session.commit()
if video_id == table_id:
logger.success(f'更新数据table_id:{table_id}')
else:
logger.success(f'新增数据table_id:{table_id}')
def scl_deploy(docker=SubscribesCleaner):
"""
@param docker: Python 类对象
@return:
"""
logger.success(f'<GeventSchedule>启动成功 -- {docker.__name__}')
def release_docker(interface: str = 'interface'):
"""
由接口解压容器主线功能
@param interface: 接口函数名
@return:
"""
logger.info(f'>> Do {docker.__name__}')
exec(f'docker().{interface}()')
schedule.every(1).minute.do(release_docker)
while True:
schedule.run_pending()
time.sleep(1)
async def load_jobs():
for job in config['jobs']:
job_type = list(job)[0]
success = JobLoader.load_job_type( job_type, job[job_type] )
if success:
logger.success(f'Loaded job: "{job[job_type]}" of type "{job_type}"')
