def run(self) -> None: # logger.warning('This is a development server. Do not use it in a production deployment.') try: for task_name in self.deploy_cluster: try: schedule.every(self.crontab['action']).minutes.do(self.push_task, task_name=task_name) if ENABLE_DDT: schedule.every(self.crontab['refresh']).minutes.do(self.rc.refresh, key_name=REDIS_SECRET_KEY.format(task_name)) logger.success(f"START DDT -- {task_name}") else: logger.warning(f'Not Authorized -- DDT({task_name})') logger.success(f"START TASK -- {task_name}/crontab:{self.crontab['action']} minutes") except schedule.IntervalError: logger.error('interval set error') self.crontab['action'] += 5 while True: schedule.run_pending() time.sleep(1) except Exception as err: logger.exception('Exception occurred ||{}'.format(err)) noticer.send_email(text_body='{}'.format(err), to='self') except KeyboardInterrupt as err: logger.stop('Forced stop ||{}'.format(err))
def push_info(self, user: dict or List[dict]): if isinstance(user, dict): user = [ user, ] elif not isinstance(user, list): logger.warning('MySQL add_user 调用格式有误') try: for user_ in user: try: sql = f'INSERT INTO v2raycs (' \ f'domain, subs, class_,end_life,res_time,passable,username,password,email,uuid) VALUES (' \ f'%s, %s, %s,%s, %s, %s,%s, %s, %s,%s)' val = (user_["domain"], user_["subs"], user_['class_'], user_['end_life'], user_["res_time"], user_['passable'], user_['username'], user_["password"], user_['email'], user_['uuid']) self.cursor.execute(sql, val) except KeyError as e: logger.warning( f"MySQL数据解析出错,user:dict必须同时包含username、password以及email的键值对{e}" ) # return 702 except pymysql.err.IntegrityError as e: logger.warning( f'{user_["username"]} -- 用户已在库,若需修改用户信息,请使用更新指令{e}') # return 701 else: logger.success(f'{user_["username"]} -- 用户添加成功') # return 700 finally: self.conn.commit() self.conn.close()
def sever_chan(title: str = None, message: str = None) -> bool: """ 调用SERVER酱微信提示 @param title: 标题最大256 @param message: 正文,支持markdown,最大64kb @return: """ if not isinstance(title, str) or not isinstance(message, str): return False import requests url = f"http://sc.ftqq.com/{SERVER_CHAN_SCKEY}.send" params = {'text': title, 'desp': message} try: res = requests.get(url, params=params) res.raise_for_status() if res.status_code == 200 and res.json().get("errmsg") == 'success': logger.success("Server酱设备通知已发送~") return True except requests.exceptions.HTTPError: logger.error( "Server酱404!!!可能原因为您的SCKEY未填写或已重置,请访问 http://sc.ftqq.com/3.version 查看解决方案" ) logger.debug('工作流将保存此漏洞数据至error.log 并继续运行,希望您常来看看……')
def push_task(self, task_name: str) -> bool: """ @param task_name: @return: """ # 输入参数的数据类型错误 if not isinstance(task_name, str): logger.error(f'The input type is wrong({task_name})') return False # 输入的参数不在模型的权限范围中 if task_name not in self.deploy_cluster: logger.error( f'Spelling error in input({task_name}),Please choose from {self.deploy_cluster}' ) return False try: # 判断缓冲队列是否已达单机采集极限 task_name = task_name.lower() self.rc_len[f'{task_name}'] = self.rc.__len__( REDIS_SECRET_KEY.format(f'{task_name}')) logger.info(f'[TEST] ||正在检查({task_name}) 任务队列...') # 若已达或超过单机采集极限,则休眠任务 if self.rc_len[f"{task_name}"] >= self.cap: logger.debug( f'[SLEEP] || 任务队列已满 ({task_name}) ({self.rc_len[f"{task_name}"]}/{self.cap})' ) return True finally: # 无论队列是否已满,执行一次ddt self.ddt(class_=task_name) try: # 执行采集任务,通过self.go决定是否启动协程加速 logger.info(f'[RUN] || ({task_name}) 采集任务启动') __task__.loads_task(task_name, self.go) # 判断任务是否完全失败,既单个类型链接的所有采集任务全部失败->Abnormal if self.rc.__len__(REDIS_SECRET_KEY.format( f'{task_name}')) < self.rc_len[f'{task_name}']: logger.error( f'[CRITICAL]Abnormal collection task({task_name})') else: return True except Exception as e: # 捕获未知错误 logger.error( f'[ERROR]{self.__class__.__name__}({task_name}) crawler engine panic {e}' ) finally: # 单个类型的链接采集结束 logger.success('[OVER] || 任务结束 {}({})'.format( self.__class__.__name__, task_name))
async def weather_report(conf): rep = WeatherReport(conf['url'],{ "tags": { "region": conf.get('region','') }, }) logger.success(f'Got "{conf["name"]:18}" report. temp: {rep.temperature}') return True
def load_any_subscribe(self, api: Chrome, element_xpath_str: str, href_xpath_str: str, class_: str, retry=0): """ @param api: ChromeDriverObject @param element_xpath_str: 用于定位链接所在的标签 @param href_xpath_str: 用于取出目标标签的属性值,既subscribe @param class_: 该subscribe类型,如`ssr`/`v2ray`/`trojan` @param retry: 失败重试 @todo 使用 retrying 模块替代retry参数实现的功能(引入断网重连,断言重试,行为回滚...) @return: """ self.subscribe = WebDriverWait(api, 30).until( EC.presence_of_element_located( (By.XPATH, element_xpath_str))).get_attribute(href_xpath_str) if self.subscribe: for x in range(3): # ['domain', 'subs', 'class_', 'end_life', 'res_time', 'passable','username', 'password', 'email'] try: domain = urlparse(self.register_url).netloc res_time = str(datetime.now(TIME_ZONE_CN)).split('.')[0] passable = 'true' docker = [ domain, self.subscribe, class_, self.end_life, res_time, passable, self.username, self.password, self.email ] FlexibleDistribute(docker=docker, at_once=self.at_once) # flexible_distribute(self.subscribe, class_, self.end_life, driver_name=self.__class__.__name__) logger.success(">> GET <{}> -> {}:{}".format( self.__class__.__name__, class_, self.subscribe)) break except Exception as e: logger.debug(">> FAILED <{}> -> {}:{}".format( self.__class__.__name__, class_, e)) time.sleep(1) continue else: return None else: if retry >= 3: raise TimeoutException retry += 1 self.load_any_subscribe(api, element_xpath_str, href_xpath_str, class_, retry)
async def shutdown(main_task=None): for task in asyncio.all_tasks(): if task == asyncio.current_task(): continue try: # pass task.cancel() try: print("task ex: ",task.exception()) except Exception as ex: pass except Exception as ex: logger.error("Task failed to cancel {}",ex) finally: logger.success(f'Finished cleanup, exiting')
def refresh(self, key_name: str) -> None: """ 原子级链接池刷新,一次性删去所有过期的key_name subscribe @param key_name:secret_key @return: """ docker: dict = self.db.hgetall(key_name) # 管理员指令获取的链接 if self.db.hlen(key_name) != 0: for subscribe, end_life in docker.items(): if self.is_stale(end_life): logger.debug(f'del-({key_name})--{subscribe}') self.db.hdel(key_name, subscribe) logger.success('{}:UPDATE - {}({})'.format(self.__class__.__name__, key_name, self.__len__(key_name))) else: logger.warning('{}:EMPTY - {}({})'.format(self.__class__.__name__, key_name, self.__len__(key_name)))
def send_email(msg, to_: List[str] or str or set, headers: str = None): """ 发送运维信息,该函数仅用于发送简单文本信息 :param msg: 正文内容 :param to_: 发送对象 1. str to_ == 'self',发送给“自己” 2. List[str] 传入邮箱列表,群发邮件(内容相同)。 :param headers: :@todo 加入日志读取功能(open file)以及富文本信息功能(html邮件) :return: 默认为'<V2Ray云彩姬>运维日志' """ headers = headers if headers else '<V2Ray云彩姬>运维日志' sender = SMTP_ACCOUNT.get('email') password = SMTP_ACCOUNT.get('sid') smtp_server = 'smtp.qq.com' message = MIMEText(msg, 'plain', 'utf-8') message['From'] = Header('ARAI.DM', 'utf-8') # 发送者 message['Subject'] = Header(f"{headers}", 'utf-8') server = smtplib.SMTP_SSL(smtp_server, 465) # 输入转换 if to_ == 'self': to_ = set(sender, ) if isinstance(to_, str): to_ = [to_, ] if isinstance(to_, list): to_ = set(to_) if not isinstance(to_, set): return False try: server.login(sender, password) for to in to_: try: message['To'] = Header(to, 'utf-8') # 接收者 server.sendmail(sender, to, message.as_string()) logger.success("发送成功->{}".format(to)) except smtplib.SMTPRecipientsRefused: logger.warning('邮箱填写错误或不存在->{}'.format(to)) except Exception as e: logger.error('>>> 发送失败 || {}'.format(e)) finally: server.quit()
def interface(self, power: int = 8) -> None: """ @param power: 协程功率 @return: """ # 任务重载 self.offload_task() # 任务启动 task_list = [] power_ = self.power if self.power else power for x in range(power_): task = gevent.spawn(self.launch) task_list.append(task) gevent.joinall(task_list) logger.success(f'<Gevent> mission completed -- <{self.__class__.__name__}>')
def callback(ch, method, properties, body): # print(1112222223333333) # print(body.decode()) d = json.loads(body.decode()) # print(66666666) vid = int(d.pop('id')) data = {} data['vid'] = vid data.update(d) filter_dict = dict(vid=data.get('vid'), for_test=int(data.get('for_test'))) video = upsert(Video, data, filter_dict) video_id = video.id session.add(video) session.flush() table_id = video.id session.commit() if video_id == table_id: logger.success(f'更新数据table_id:{table_id}') else: logger.success(f'新增数据table_id:{table_id}')
def scl_deploy(docker=SubscribesCleaner): """ @param docker: Python 类对象 @return: """ logger.success(f'<GeventSchedule>启动成功 -- {docker.__name__}') def release_docker(interface: str = 'interface'): """ 由接口解压容器主线功能 @param interface: 接口函数名 @return: """ logger.info(f'>> Do {docker.__name__}') exec(f'docker().{interface}()') schedule.every(1).minute.do(release_docker) while True: schedule.run_pending() time.sleep(1)
async def load_jobs(): for job in config['jobs']: job_type = list(job)[0] success = JobLoader.load_job_type( job_type, job[job_type] ) if success: logger.success(f'Loaded job: "{job[job_type]}" of type "{job_type}"')