def __init__(self): self.running = False class DB(object): user = db.UserDB() tpl = db.TPLDB() task = db.TaskDB() tasklog = db.TaskLogDB() self.db = DB self.fetcher = Fetcher()
def __init__(self): if config.db_type == 'sqlite3': import sqlite3_db as db else: import db class DB(object): user = db.UserDB() tpl = db.TPLDB() task = db.TaskDB() tasklog = db.TaskLogDB() site = db.SiteDB() pubtpl = db.PubTplDB() self.db = DB self.fetcher = Fetcher()
def __init__(self): self.running = False if config.db_type == 'sqlite3': import sqlite3_db as db else: import db class DB(object): user = db.UserDB() tpl = db.TPLDB() task = db.TaskDB() tasklog = db.TaskLogDB() self.db = DB self.fetcher = Fetcher()
def __init__(self): TaskBase.__init__(self) self.resultCount = 0 self.fetcher = Fetcher(m_settings.mfile_database()) self.beanstalkclient = m_settings.beanstalk_client() self.logger = getLogger(self.__class__.__name__, console_out=False, level="debug") self.province = "安徽省" self.city = "合肥"
def __init__(self): TaskBase.__init__(self) self.resultCount = 0 self.fetcher = Fetcher(m_settings.mfile_database()) self.beans_client = m_settings.beanstalk_client() self.logger = getLogger(self.__class__.__name__, console_out=False, level="debug") self.province = "湖南 " self.city = "" self.all_ip = staticproxy.get_all_proxie() if not isinstance(self.all_ip, list) or len(self.all_ip) <= 0: raise Exception('代理初始化异常。。。')
def __init__(self): settings = dict( template_path = os.path.join(os.path.dirname(__file__), "tpl"), static_path = os.path.join(os.path.dirname(__file__), "static"), debug = config.debug, gzip = config.gzip, autoreload = config.autoreload, cookie_secret = config.cookie_secret, login_url = '/login', ) version_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), "version.json") super(Application, self).__init__(handlers, **settings) self.jinja_env = jinja2.Environment( loader=jinja2.FileSystemLoader(settings['template_path']), extensions=['jinja2.ext.autoescape', 'jinja2.ext.loopcontrols', ], autoescape=True, auto_reload=config.autoreload) if config.db_type == 'sqlite3': import sqlite3_db as db else: import db class DB(object): user = db.UserDB() tpl = db.TPLDB() task = db.TaskDB() tasklog = db.TaskLogDB() push_request = db.PRDB() redis = db.RedisDB() site = db.SiteDB() pubtpl = db.PubTplDB() self.db = DB self.fetcher = Fetcher() with open(version_path, "r", encoding='utf-8') as f: version_data = json.load(f) self.jinja_env.globals.update({ 'config': config, 'format_date': utils.format_date, 'varbinary2ip': utils.varbinary2ip, 'version': version_data['version'] }) self.jinja_env.filters.update(ui_methods)
class MainWorker(object): def __init__(self): self.running = False if config.db_type == 'sqlite3': import sqlite3_db as db else: import db class DB(object): user = db.UserDB() tpl = db.TPLDB() task = db.TaskDB() tasklog = db.TaskLogDB() site = db.SiteDB() pubtpl = db.PubTplDB() self.db = DB self.fetcher = Fetcher() def __call__(self): # self.running = tornado.ioloop.IOLoop.current().spawn_callback(self.run) # if self.running: # success, failed = self.running # if success or failed: # logger.info('%d task done. %d success, %d failed' % (success+failed, success, failed)) if self.running: return self.running = self.run() def done(future): self.running = None success, failed = future.result() if success or failed: logger.info('%d task done. %d success, %d failed' % (success+failed, success, failed)) return self.running.add_done_callback(done) def ClearLog(self, taskid): logDay = int(self.db.site.get(1, fields=('logDay'))['logDay']) for log in self.db.tasklog.list(taskid = taskid, fields=('id', 'ctime')): if (time.time() - log['ctime']) > (logDay * 24 * 60 * 60): self.db.tasklog.delete(log['id']) async def push_batch(self): try: userlist = await asyncio.wait_for(asyncio.get_event_loop().run_in_executor(None, functools.partial(self.db.user.list,fields=('id', 'email', 'status', 'push_batch'))),timeout=3.0) pushtool = pusher() logging.debug('scaned push_batch task, waiting...') if userlist: for user in userlist: userid = user['id'] push_batch = json.loads(user['push_batch']) if user['status'] == "Enable" and push_batch["sw"] and isinstance(push_batch['time'],(float,int)) and time.time() >= push_batch['time']: title = u"定期签到日志推送" delta = push_batch.get("delta", 86400) logtemp = "{}".format(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(push_batch['time']))) tmpdict = {} tmp = "" numlog = 0 task_list = await asyncio.wait_for(asyncio.get_event_loop().run_in_executor(None, functools.partial(self.db.task.list,userid=userid, fields=('id', 'tplid', 'note', 'disabled', 'last_success', 'last_failed', 'pushsw'))),timeout=3.0) for task in task_list: pushsw = json.loads(task['pushsw']) if pushsw["pushen"] and (task["disabled"] == 0 or (task.get("last_success", 0) and task.get("last_success", 0) >= push_batch['time']-delta) or (task.get("last_failed", 0) and task.get("last_failed", 0) >= push_batch['time']-delta)): tmp0 = "" tasklog_list = await asyncio.wait_for(asyncio.get_event_loop().run_in_executor(None, functools.partial(self.db.tasklog.list,taskid=task["id"], fields=('success', 'ctime', 'msg'))),timeout=3.0) for log in tasklog_list: if (push_batch['time'] - delta) < log['ctime'] <= push_batch['time']: tmp0 += "\\r\\n时间: {}\\r\\n日志: {}".format(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(log['ctime'])), log['msg']) numlog += 1 tmplist = tmpdict.get(task['tplid'],[]) if tmp0: tmplist.append("\\r\\n-----任务{0}-{1}-----{2}\\r\\n".format(len(tmplist)+1, task['note'], tmp0)) else: tmplist.append("\\r\\n-----任务{0}-{1}-----\\r\\n记录期间未执行签到,请检查任务! \\r\\n".format(len(tmplist)+1, task['note'])) tmpdict[task['tplid']] = tmplist for tmpkey in tmpdict: tmp = "\\r\\n\\r\\n=====签到: {0}=====".format(self.db.tpl.get(tmpkey, fields=('sitename'))['sitename']) tmp += ''.join(tmpdict[tmpkey]) logtemp += tmp push_batch["time"] = push_batch['time'] + delta self.db.user.mod(userid, push_batch=json.dumps(push_batch)) if tmp and numlog: user_email = user.get('email','Unkown') logger.debug("Start push batch log for {}".format(user_email)) await pushtool.pusher(userid, {"pushen": bool(push_batch.get("sw",False))}, 4080, title, logtemp) logger.info("Success push batch log for {}".format(user_email)) except Exception as e: traceback.print_exc() logger.error('push batch task failed: {}'.format(str(e))) @gen.coroutine def run(self): running = [] success = 0 failed = 0 try: for task in self.scan(): running.append(self.do(task)) if len(running) > 50: logging.debug('scaned %d task, waiting...', len(running)) result = yield running[:10] for each in result: if each: success += 1 else: failed += 1 running = running[10:] logging.debug('scaned %d task, waiting...', len(running)) result = yield running for each in result: if each: success += 1 else: failed += 1 if config.push_batch_sw: yield self.push_batch() except Exception as e: logging.exception(e) return (success, failed) scan_fields = ('id', 'tplid', 'userid', 'init_env', 'env', 'session', 'retry_count', 'retry_interval', 'last_success', 'last_failed', 'success_count', 'failed_count', 'last_failed_count', 'next', 'disabled', ) def scan(self): return self.db.task.scan(fields=self.scan_fields) @staticmethod def failed_count_to_time(last_failed_count, retry_count=8, retry_interval=None, interval=None): next = None if last_failed_count < retry_count or retry_count == -1: if retry_interval: next = retry_interval else: if last_failed_count == 0: next = 10 * 60 elif last_failed_count == 1: next = 110 * 60 elif last_failed_count == 2: next = 4 * 60 * 60 elif last_failed_count == 3: next = 6 * 60 * 60 elif last_failed_count < retry_count or retry_count == -1: next = 11 * 60 * 60 else: next = None elif retry_count == 0: next = None if interval is None: interval = 24 * 60 * 60 if next: next = min(next, interval / 2) return next @staticmethod def fix_next_time(next, gmt_offset=-8*60): date = datetime.datetime.utcfromtimestamp(next) local_date = date - datetime.timedelta(minutes=gmt_offset) if local_date.hour < 2: next += 2 * 60 * 60 if local_date.hour > 21: next -= 3 * 60 * 60 return next @staticmethod def is_tommorrow(next, gmt_offset=-8*60): date = datetime.datetime.utcfromtimestamp(next) now = datetime.datetime.utcnow() local_date = date - datetime.timedelta(minutes=gmt_offset) local_now = now - datetime.timedelta(minutes=gmt_offset) local_tomorrow = local_now + datetime.timedelta(hours=24) if local_date.day == local_tomorrow.day and not now.hour > 22: return True elif local_date.hour > 22: return True else: return False async def do(self, task): task['note'] = self.db.task.get(task['id'], fields=('note'))['note'] user = self.db.user.get(task['userid'], fields=('id', 'email', 'email_verified', 'nickname', 'logtime')) tpl = self.db.tpl.get(task['tplid'], fields=('id', 'userid', 'sitename', 'siteurl', 'tpl', 'interval', 'last_success')) ontime = self.db.task.get(task['id'], fields=('ontime', 'ontimeflg', 'pushsw', 'newontime', 'next')) newontime = json.loads(ontime["newontime"]) pushtool = pusher() caltool = cal() logtime = json.loads(user['logtime']) pushsw = json.loads(ontime['pushsw']) if 'ErrTolerateCnt' not in logtime:logtime['ErrTolerateCnt'] = 0 if task['disabled']: self.db.tasklog.add(task['id'], False, msg='task disabled.') self.db.task.mod(task['id'], next=None, disabled=1) return False if not user: self.db.tasklog.add(task['id'], False, msg='no such user, disabled.') self.db.task.mod(task['id'], next=None, disabled=1) return False if not tpl: self.db.tasklog.add(task['id'], False, msg='tpl missing, task disabled.') self.db.task.mod(task['id'], next=None, disabled=1) return False if tpl['userid'] and tpl['userid'] != user['id']: self.db.tasklog.add(task['id'], False, msg='no permission error, task disabled.') self.db.task.mod(task['id'], next=None, disabled=1) return False start = time.time() try: fetch_tpl = self.db.user.decrypt(0 if not tpl['userid'] else task['userid'], tpl['tpl']) env = dict( variables = self.db.user.decrypt(task['userid'], task['init_env']), session = [], ) url = utils.parse_url(env['variables'].get('_proxy')) if not url: new_env = await gen.convert_yielded(self.fetcher.do_fetch(fetch_tpl, env)) else: proxy = { 'scheme': url['scheme'], 'host': url['host'], 'port': url['port'], 'username': url['username'], 'password': url['password'] } new_env = await gen.convert_yielded(self.fetcher.do_fetch(fetch_tpl, env, [proxy])) variables = self.db.user.encrypt(task['userid'], new_env['variables']) session = self.db.user.encrypt(task['userid'], new_env['session'].to_json() if hasattr(new_env['session'], 'to_json') else new_env['session']) # todo next not mid night if (newontime['sw']): if ('mode' not in newontime): newontime['mode'] = 'ontime' if (newontime['mode'] == 'ontime'): newontime['date'] = (datetime.datetime.now()+datetime.timedelta(days=1)).strftime("%Y-%m-%d") next = caltool.calNextTs(newontime)['ts'] else: next = time.time() + max((tpl['interval'] if tpl['interval'] else 24 * 60 * 60), 1*60) if tpl['interval'] is None: next = self.fix_next_time(next) # success feedback self.db.tasklog.add(task['id'], success=True, msg=new_env['variables'].get('__log__')) self.db.task.mod(task['id'], last_success=time.time(), last_failed_count=0, success_count=task['success_count']+1, env=variables, session=session, mtime=time.time(), next=next) self.db.tpl.incr_success(tpl['id']) t = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') title = u"签到任务 {0}-{1} 成功".format(tpl['sitename'], task['note']) logtemp = new_env['variables'].get('__log__') logtemp = u"{0} \\r\\n日志:{1}".format(t, logtemp) await pushtool.pusher(user['id'], pushsw, 0x2, title, logtemp) logger.info('taskid:%d tplid:%d successed! %.5fs', task['id'], task['tplid'], time.time()-start) # delete log self.ClearLog(task['id']) except Exception as e: # failed feedback traceback.print_exc() next_time_delta = self.failed_count_to_time(task['last_failed_count'], task['retry_count'], task['retry_interval'], tpl['interval']) t = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') title = u"签到任务 {0}-{1} 失败".format(tpl['sitename'], task['note']) content = u"{0} \\r\\n日志:{1}".format(t, str(e)) disabled = False if next_time_delta: next = time.time() + next_time_delta content = content + u" \\r\\n下次运行时间:{0}".format(time.strftime("%Y-%m-%d %H:%M:%S",time.localtime(next))) if (logtime['ErrTolerateCnt'] <= task['last_failed_count']): await pushtool.pusher(user['id'], pushsw, 0x1, title, content) else: disabled = True next = None content = u" \\r\\n任务已禁用" await pushtool.pusher(user['id'], pushsw, 0x1, title, content) self.db.tasklog.add(task['id'], success=False, msg=str(e)) self.db.task.mod(task['id'], last_failed=time.time(), failed_count=task['failed_count']+1, last_failed_count=task['last_failed_count']+1, disabled = disabled, mtime = time.time(), next=next) self.db.tpl.incr_failed(tpl['id']) logger.error('taskid:%d tplid:%d failed! %r %.4fs', task['id'], task['tplid'], str(e), time.time()-start) return False return True
class MainWorker(object): def __init__(self): self.running = False if config.db_type == 'sqlite3': import sqlite3_db as db else: import db class DB(object): user = db.UserDB() tpl = db.TPLDB() task = db.TaskDB() tasklog = db.TaskLogDB() site = db.SiteDB() pubtpl = db.PubTplDB() self.db = DB self.fetcher = Fetcher() def __call__(self): if self.running: return self.running = self.run() def done(future): self.running = None success, failed = future.result() if success or failed: logger.info('%d task done. %d success, %d failed' % (success + failed, success, failed)) return self.running.add_done_callback(done) def ClearLog(self, taskid): logDay = int(self.db.site.get(1, fields=('logDay'))['logDay']) for log in self.db.tasklog.list(taskid=taskid, fields=('id', 'ctime')): if (time.time() - log['ctime']) > (logDay * 24 * 60 * 60): self.db.tasklog.delete(log['id']) @gen.coroutine def run(self): running = [] success = 0 failed = 0 try: for task in self.scan(): running.append(self.do(task)) if len(running) > 50: logging.debug('scaned %d task, waiting...', len(running)) result = yield running[:10] for each in result: if each: success += 1 else: failed += 1 running = running[10:] logging.debug('scaned %d task, waiting...', len(running)) result = yield running for each in result: if each: success += 1 else: failed += 1 except Exception as e: logging.exception(e) raise gen.Return((success, failed)) scan_fields = ( 'id', 'tplid', 'userid', 'init_env', 'env', 'session', 'last_success', 'last_failed', 'success_count', 'failed_count', 'last_failed_count', 'next', 'disabled', ) def scan(self): return self.db.task.scan(fields=self.scan_fields) @staticmethod def failed_count_to_time(last_failed_count, interval=None): if last_failed_count == 0: next = 10 * 60 elif last_failed_count == 1: next = 110 * 60 elif last_failed_count == 2: next = 240 * 60 elif last_failed_count == 3: next = 360 * 60 elif last_failed_count < 8: next = 11 * 60 * 60 else: next = None if interval is None: interval = 24 * 60 * 60 if next and next > interval / 2: next = interval / 2 return next @staticmethod def fix_next_time(next, gmt_offset=-8 * 60): date = datetime.datetime.utcfromtimestamp(next) local_date = date - datetime.timedelta(minutes=gmt_offset) if local_date.hour < 2: next += 2 * 60 * 60 if local_date.hour > 21: next -= 3 * 60 * 60 return next @staticmethod def is_tommorrow(next, gmt_offset=-8 * 60): date = datetime.datetime.utcfromtimestamp(next) now = datetime.datetime.utcnow() local_date = date - datetime.timedelta(minutes=gmt_offset) local_now = now - datetime.timedelta(minutes=gmt_offset) local_tomorrow = local_now + datetime.timedelta(hours=24) if local_date.day == local_tomorrow.day and not now.hour > 22: return True elif local_date.hour > 22: return True else: return False @gen.coroutine def do(self, task): task['note'] = self.db.task.get(task['id'], fields=('note'))['note'] user = self.db.user.get(task['userid'], fields=('id', 'email', 'email_verified', 'nickname', 'logtime')) tpl = self.db.tpl.get(task['tplid'], fields=('id', 'userid', 'sitename', 'siteurl', 'tpl', 'interval', 'last_success')) ontime = self.db.task.get(task['id'], fields=('ontime', 'ontimeflg', 'pushsw', 'newontime', 'next')) newontime = json.loads(ontime["newontime"]) pushtool = pusher() caltool = cal() logtime = json.loads(user['logtime']) pushsw = json.loads(ontime['pushsw']) if 'ErrTolerateCnt' not in logtime: logtime['ErrTolerateCnt'] = 0 if task['disabled']: self.db.tasklog.add(task['id'], False, msg='task disabled.') self.db.task.mod(task['id'], next=None, disabled=1) raise gen.Return(False) if not user: self.db.tasklog.add(task['id'], False, msg='no such user, disabled.') self.db.task.mod(task['id'], next=None, disabled=1) raise gen.Return(False) if not tpl: self.db.tasklog.add(task['id'], False, msg='tpl missing, task disabled.') self.db.task.mod(task['id'], next=None, disabled=1) raise gen.Return(False) if tpl['userid'] and tpl['userid'] != user['id']: self.db.tasklog.add(task['id'], False, msg='no permission error, task disabled.') self.db.task.mod(task['id'], next=None, disabled=1) raise gen.Return(False) start = time.time() try: fetch_tpl = self.db.user.decrypt( 0 if not tpl['userid'] else task['userid'], tpl['tpl']) env = dict( variables=self.db.user.decrypt(task['userid'], task['init_env']), session=[], ) url = utils.parse_url(env['variables'].get('_proxy')) if not url: new_env = yield self.fetcher.do_fetch(fetch_tpl, env) else: proxy = { 'host': url['host'], 'port': url['port'], } new_env = yield self.fetcher.do_fetch(fetch_tpl, env, [proxy]) variables = self.db.user.encrypt(task['userid'], new_env['variables']) session = self.db.user.encrypt( task['userid'], new_env['session'].to_json() if hasattr( new_env['session'], 'to_json') else new_env['session']) # todo next not mid night if (newontime['sw']): if ('mode' not in newontime): newontime['mode'] = 'ontime' if (newontime['mode'] == 'ontime'): newontime['date'] = ( datetime.datetime.now() + datetime.timedelta(days=1)).strftime("%Y-%m-%d") next = caltool.calNextTs(newontime)['ts'] else: next = time.time() + max( (tpl['interval'] if tpl['interval'] else 24 * 60 * 60), 1 * 60) if tpl['interval'] is None: next = self.fix_next_time(next) # success feedback self.db.tasklog.add(task['id'], success=True, msg=new_env['variables'].get('__log__')) self.db.task.mod(task['id'], last_success=time.time(), last_failed_count=0, success_count=task['success_count'] + 1, env=variables, session=session, mtime=time.time(), next=next) self.db.tpl.incr_success(tpl['id']) t = datetime.datetime.now().strftime('%m-%d %H:%M:%S') title = u"签到任务 {0}-{1} 成功".format(tpl['sitename'], task['note']) logtemp = new_env['variables'].get('__log__') logtemp = u"{0} 日志:{1}".format(t, logtemp) pushtool.pusher(user['id'], pushsw, 0x2, title, logtemp) logger.info('taskid:%d tplid:%d successed! %.4fs', task['id'], task['tplid'], time.time() - start) # delete log self.ClearLog(task['id']) except Exception as e: # failed feedback next_time_delta = self.failed_count_to_time( task['last_failed_count'], tpl['interval']) t = datetime.datetime.now().strftime('%m-%d %H:%M:%S') title = u"签到任务 {0}-{1} 失败".format(tpl['sitename'], task['note']) content = u"日志:{log}".format(log=e) disabled = False if next_time_delta: next = time.time() + next_time_delta content = content + u"下次运行时间:{0}".format( time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(next))) if (logtime['ErrTolerateCnt'] <= task['last_failed_count']): pushtool.pusher(user['id'], pushsw, 0x1, title, content) else: disabled = True next = None content = u"任务已禁用" pushtool.pusher(user['id'], pushsw, 0x1, title, content) self.db.tasklog.add(task['id'], success=False, msg=unicode(e)) self.db.task.mod(task['id'], last_failed=time.time(), failed_count=task['failed_count'] + 1, last_failed_count=task['last_failed_count'] + 1, disabled=disabled, mtime=time.time(), next=next) self.db.tpl.incr_failed(tpl['id']) logger.error('taskid:%d tplid:%d failed! %r %.4fs', task['id'], task['tplid'], e, time.time() - start) raise gen.Return(False) raise gen.Return(True)
class pusher(object): def __init__(self): if config.db_type == 'sqlite3': import sqlite3_db as db else: import db class DB(object): user = db.UserDB() tpl = db.TPLDB() task = db.TaskDB() tasklog = db.TaskLogDB() site = db.SiteDB() pubtpl = db.PubTplDB() self.db = DB self.fetcher = Fetcher() async def pusher(self, userid, pushsw, flg, title, content): notice = self.db.user.get(userid, fields=('skey', 'barkurl', 'noticeflg', 'wxpusher', 'qywx_token', 'tg_token', 'dingding_token', 'diypusher')) if (notice['noticeflg'] & flg != 0): user = self.db.user.get(userid, fields=('id', 'email', 'email_verified', 'nickname')) diypusher = notice['diypusher'] if (diypusher != ''):diypusher = json.loads(diypusher) self.barklink = notice['barkurl'] pusher = {} pusher["mailpushersw"] = False if (notice['noticeflg'] & 0x80) == 0 else True pusher["barksw"] = False if (notice['noticeflg'] & 0x40) == 0 else True pusher["schansw"] = False if (notice['noticeflg'] & 0x20) == 0 else True pusher["wxpushersw"] = False if (notice['noticeflg'] & 0x10) == 0 else True pusher["cuspushersw"] = False if (notice['noticeflg'] & 0x100) == 0 else True pusher["qywxpushersw"] = False if (notice['noticeflg'] & 0x200) == 0 else True pusher["tgpushersw"] = False if (notice['noticeflg'] & 0x400) == 0 else True pusher["dingdingpushersw"] = False if (notice['noticeflg'] & 0x800) == 0 else True def nonepush(*args): return if (pushsw['pushen']): send2bark = self.send2bark if (pusher["barksw"]) else nonepush send2s = self.send2s if (pusher["schansw"]) else nonepush send2wxpusher = self.send2wxpusher if (pusher["wxpushersw"]) else nonepush sendmail = self.sendmail if (pusher["mailpushersw"]) else nonepush cus_pusher_send = self.cus_pusher_send if (pusher["cuspushersw"]) else nonepush qywx_pusher_send = self.qywx_pusher_send if (pusher["qywxpushersw"]) else nonepush send2tg = self.send2tg if (pusher["tgpushersw"]) else nonepush send2dingding = self.send2dingding if (pusher["dingdingpushersw"]) else nonepush if (pushsw['pushen']): await gen.convert_yielded([send2bark(notice['barkurl'], title, content), send2s(notice['skey'], title, content), send2wxpusher( notice['wxpusher'], title+u" "+content), sendmail( user['email'], title, content), cus_pusher_send( diypusher, title, content), qywx_pusher_send( notice['qywx_token'], title, content), send2tg( notice['tg_token'], title, content), send2dingding(notice['dingding_token'], title, content)]) async def send2bark(self, barklink, title, content): r = 'False' try: link = barklink if (link[-1] != '/'): link=link+'/' content = content.replace('\\r\\n','\n') d = {"title":title,"body":content} obj = {'request': {'method': 'POST', 'url': link, 'headers': [{'name' : 'Content-Type', 'value': 'application/json; charset=UTF-8'}], 'cookies': [], 'data':json.dumps(d)}, 'rule': { 'success_asserts': [], 'failed_asserts': [], 'extract_variables': []}, 'env': {'variables': {}, 'session': []}} _,_,res = await gen.convert_yielded(self.fetcher.build_response(obj = obj)) r = 'True' except Exception as e: r = traceback.format_exc() print(r) return r async def send2s(self, skey, title, content): r = 'False' if (skey != ""): try: link = u"https://sctapi.ftqq.com/{0}.send".format(skey.replace(".send", "")) content = content.replace('\\r\\n','\n\n') d = {'text': title, 'desp': content} obj = {'request': {'method': 'POST', 'url': link, 'headers': [{'name' : 'Content-Type', 'value': 'application/x-www-form-urlencoded; charset=UTF-8'}], 'cookies': [], 'data':utils.urllib.parse.urlencode(d)}, 'rule': { 'success_asserts': [], 'failed_asserts': [], 'extract_variables': []}, 'env': {'variables': {}, 'session': []}} _,_,res = await gen.convert_yielded(self.fetcher.build_response(obj = obj)) r = 'True' except Exception as e: r = traceback.format_exc() print(r) return r async def send2tg(self, tg_token, title, content): r = 'False' tmp = tg_token.split(';') tgToken = '' tgUserId = '' if len(tmp) >= 2: tgToken = tmp[0] tgUserId = tmp[1] tgHost = tmp[2] if len(tmp) >= 3 else '' proxy = utils.parse_url(tmp[3]) if len(tmp) >= 4 else '' pic = tmp[4] if len(tmp) >= 5 else '' if tgToken and tgUserId: try: token = tgToken chat_id = tgUserId #TG_BOT的token #token = os.environ.get('TG_TOKEN') #用户的ID #chat_id = os.environ.get('TG_USERID') if not tgHost: link = u'https://api.telegram.org/bot{0}/sendMessage'.format(token) else: if tgHost[-1]!='/': tgHost = tgHost + '/' if 'http://' in tgHost or 'https://' in tgHost: link = u'{0}bot{1}/sendMessage'.format(tgHost,token) else: link = u'https://{0}bot{1}/sendMessage'.format(tgHost,token) picurl = config.push_pic if pic == '' else pic content = content.replace('\\r\\n','</pre>\n<pre>') d = {'chat_id': str(chat_id), 'text': '<b>' + title + '</b>' + '\n<pre>' + content + '</pre>\n' + '------<a href="' + picurl + '">QianDao提醒</a>------', 'disable_web_page_preview':'false', 'parse_mode': 'HTML'} obj = {'request': {'method': 'POST', 'url': link, 'headers': [{'name' : 'Content-Type', 'value': 'application/json; charset=UTF-8'}], 'cookies': [], 'data':json.dumps(d)}, 'rule': { 'success_asserts': [], 'failed_asserts': [], 'extract_variables': []}, 'env': {'variables': {}, 'session': []}} if proxy: _,_,res = await gen.convert_yielded(self.fetcher.build_response(obj = obj, proxy = proxy)) else: _,_,res = await gen.convert_yielded(self.fetcher.build_response(obj = obj)) r = 'True' except Exception as e: r = traceback.format_exc() print(r) return r async def send2dingding(self, dingding_token, title, content): r = 'False' tmp = dingding_token.split(';') if len(tmp) >= 1: dingding_token = tmp[0] pic = tmp[1] if len(tmp) >= 2 else '' if (dingding_token != ""): try: link = u"https://oapi.dingtalk.com/robot/send?access_token={0}".format(dingding_token) picurl = config.push_pic if pic == '' else pic content = content.replace('\\r\\n','\n\n > ') d = {"msgtype":"markdown","markdown":{"title":title,"text":"![QianDao](" + picurl + ")\n " + "#### "+ title + "\n > " +content}} obj = {'request': {'method': 'POST', 'url': link, 'headers': [{'name' : 'Content-Type', 'value': 'application/json; charset=UTF-8'}], 'cookies': [], 'data':json.dumps(d)}, 'rule': { 'success_asserts': [], 'failed_asserts': [], 'extract_variables': []}, 'env': {'variables': {}, 'session': []}} _,_,res = await gen.convert_yielded(self.fetcher.build_response(obj = obj)) r = 'True' except Exception as e: r = traceback.format_exc() print(r) return r async def send2wxpusher(self, wxpusher, content): r = 'False' temp = wxpusher.split(";") wxpusher_token = temp[0] if (len(temp) >= 2) else "" wxpusher_uid = temp[1] if (len(temp) >= 2) else "" if (wxpusher_token != "") and (wxpusher_uid != ""): try: link = "http://wxpusher.zjiecode.com/api/send/message" content = content.replace('\\r\\n','\n') d = { "appToken":wxpusher_token, "content":content, "contentType":3, "uids":[ wxpusher_uid ] } obj = {'request': {'method': 'POST', 'url': link, 'headers': [{'name' : 'Content-Type', 'value': 'application/json; charset=UTF-8'}], 'cookies': [], 'data':json.dumps(d)}, 'rule': { 'success_asserts': [], 'failed_asserts': [], 'extract_variables': []}, 'env': {'variables': {}, 'session': []}} _,_,res = await gen.convert_yielded(self.fetcher.build_response(obj = obj)) r = 'True' except Exception as e: r = traceback.format_exc() print(r) return r async def cus_pusher_send(self, diypusher, t, log): r = 'False' try: curltmp = diypusher['curl'].format(log=log, t=t) if (diypusher['headers']): headerstmp = json.loads(diypusher['headers'].replace('{log}', log).replace("{t}", t)) else: headerstmp = {} if (diypusher['mode'] == 'POST'): postDatatmp = diypusher['postData'].replace('{log}', log).replace("{t}", t).replace("\\r\\n","\r\n" ) if (postDatatmp != ''): postDatatmp = json.loads(postDatatmp) if headerstmp: headerstmp.pop('content-type','') headerstmp.pop('Content-Type','') if (diypusher['postMethod'] == 'x-www-form-urlencoded'): headerstmp['Content-Type'] = "application/x-www-form-urlencoded; charset=UTF-8" headerstmp = [{'name': name, 'value': headerstmp[name]} for name in headerstmp] obj = {'request': {'method': 'POST', 'url': curltmp, 'headers': headerstmp, 'cookies': [], 'data':utils.urllib.parse.urlencode(postDatatmp)}, 'rule': { 'success_asserts': [], 'failed_asserts': [], 'extract_variables': []}, 'env': {'variables': {}, 'session': []}} else: headerstmp['Content-Type'] = "application/json; charset=UTF-8" headerstmp = [{'name': name, 'value': headerstmp[name]} for name in headerstmp] obj = {'request': {'method': 'POST', 'url': curltmp, 'headers': headerstmp, 'cookies': [], 'data':json.dumps(postDatatmp)}, 'rule': { 'success_asserts': [], 'failed_asserts': [], 'extract_variables': []}, 'env': {'variables': {}, 'session': []}} elif (diypusher['mode'] == 'GET'): headerstmp = [{'name': name, 'value': headerstmp[name]} for name in headerstmp] obj = {'request': {'method': 'GET', 'url': curltmp, 'headers': headerstmp, 'cookies': []}, 'rule': { 'success_asserts': [], 'failed_asserts': [], 'extract_variables': []}, 'env': {'variables': {}, 'session': []}} else: raise Exception(u'模式未选择') _,_,res = await gen.convert_yielded(self.fetcher.build_response(obj = obj)) if (res.code == 200): r = "True" except Exception as e: r = traceback.format_exc() return r async def qywx_pusher_send(self, qywx_token, t, log): r = 'False' try: qywx = {} tmp = qywx_token.split(';') if len(tmp) >= 3: qywx[u'企业ID'] = tmp[0] qywx[u'应用ID'] = tmp[1] qywx[u'应用密钥'] = tmp[2] qywx[u'图片'] = tmp[3] if len(tmp) >= 4 else '' else: raise Exception(u'企业微信token错误') access_url = 'https://qyapi.weixin.qq.com/cgi-bin/gettoken?corpid={id}&corpsecret={secret}'.format(id=qywx[u'企业ID'], secret=qywx[u'应用密钥']) obj = {'request': {'method': 'GET', 'url': access_url, 'headers': [], 'cookies': []}, 'rule': { 'success_asserts': [], 'failed_asserts': [], 'extract_variables': []}, 'env': {'variables': {}, 'session': []}} _,_,res = await gen.convert_yielded(self.fetcher.build_response(obj = obj)) get_access_token_res = json.loads(res.body) if (get_access_token_res['access_token'] != '' and get_access_token_res['errmsg'] == 'ok'): msgUrl = 'https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token={0}'.format(get_access_token_res['access_token']) postData = {"touser" : "@all", "msgtype" : "news", "agentid" : qywx[u'应用ID'], "news" : { "articles" : [ { "title" : t, "description" : log.replace("\\r\\n","\n" ), "url" : "", "picurl" : config.push_pic if qywx[u'图片'] == '' else qywx[u'图片'] } ] } } obj = {'request': {'method': 'POST', 'url': msgUrl, 'headers': [{'name' : 'Content-Type', 'value': 'application/json; charset=UTF-8'}], 'cookies': [], 'data':json.dumps(postData)}, 'rule': { 'success_asserts': [], 'failed_asserts': [], 'extract_variables': []}, 'env': {'variables': {}, 'session': []}} _,_,msg_res = await gen.convert_yielded(self.fetcher.build_response(obj = obj)) tmp = json.loads(msg_res.body) if (tmp['errmsg'] == 'ok' and tmp['errcode'] == 0): r = 'True' except Exception as e: r = traceback.format_exc() print(r) return r async def sendmail(self, email, title, content): user = self.db.user.get(email=email, fields=('id', 'email', 'email_verified', 'nickname')) if user['email'] and user['email_verified']: try: content = content.replace('\\r\\n','\n') await gen.convert_yielded(utils.send_mail(to = email, subject = u"在网站{0} {1}".format(config.domain, title), text = content, shark=True)) except Exception as e: logging.error('tasend mail error: %r', e)
class MainWorker(object): def __init__(self): self.running = False class DB(object): user = db.UserDB() tpl = db.TPLDB() task = db.TaskDB() tasklog = db.TaskLogDB() self.db = DB self.fetcher = Fetcher() def __call__(self): if self.running: return self.running = self.run() def done(future): self.running = None success, failed = future.result() if success or failed: logger.info("%d task done. %d success, %d failed" % (success + failed, success, failed)) return self.running.add_done_callback(done) @gen.coroutine def run(self): running = [] success = 0 failed = 0 try: for task in self.scan(): running.append(self.do(task)) if len(running) > 50: logging.debug("scaned %d task, waiting...", len(running)) result = yield running[:10] for each in result: if each: success += 1 else: failed += 1 running = running[10:] logging.debug("scaned %d task, waiting...", len(running)) result = yield running for each in result: if each: success += 1 else: failed += 1 except Exception as e: logging.exception(e) raise gen.Return((success, failed)) scan_fields = ( "id", "tplid", "userid", "init_env", "env", "session", "last_success", "last_failed", "success_count", "failed_count", "last_failed_count", "next", "disabled", ) def scan(self): return self.db.task.scan(fields=self.scan_fields) @staticmethod def failed_count_to_time(last_failed_count, interval=None): if last_failed_count == 0: next = 10 * 60 elif last_failed_count == 1: next = 110 * 60 elif last_failed_count == 2: next = 240 * 60 elif last_failed_count == 3: next = 360 * 60 elif last_failed_count < 8: next = 11 * 60 * 60 else: next = None if interval is None: interval = 24 * 60 * 60 if next and next > interval / 2: next = interval / 2 return next @staticmethod def fix_next_time(next, gmt_offset=-8 * 60): date = datetime.datetime.utcfromtimestamp(next) local_date = date - datetime.timedelta(minutes=gmt_offset) if local_date.hour < 2: next += 2 * 60 * 60 if local_date.hour > 21: next -= 3 * 60 * 60 return next @staticmethod def is_tommorrow(next, gmt_offset=-8 * 60): date = datetime.datetime.utcfromtimestamp(next) now = datetime.datetime.utcnow() local_date = date - datetime.timedelta(minutes=gmt_offset) local_now = now - datetime.timedelta(minutes=gmt_offset) local_tomorrow = local_now + datetime.timedelta(hours=24) if local_date.day == local_tomorrow.day and not now.hour > 22: return True elif local_date.hour > 22: return True else: return False @gen.coroutine def do(self, task): user = self.db.user.get(task["userid"], fields=("id", "email", "email_verified", "nickname")) tpl = self.db.tpl.get( task["tplid"], fields=("id", "userid", "sitename", "siteurl", "tpl", "interval", "last_success") ) if task["disabled"]: self.db.tasklog.add(task["id"], False, msg="task disabled.") self.db.task.mod(task["id"], next=None, disabled=1) raise gen.Return(False) if not user: self.db.tasklog.add(task["id"], False, msg="no such user, disabled.") self.db.task.mod(task["id"], next=None, disabled=1) raise gen.Return(False) if not tpl: self.db.tasklog.add(task["id"], False, msg="tpl missing, task disabled.") self.db.task.mod(task["id"], next=None, disabled=1) raise gen.Return(False) if tpl["userid"] and tpl["userid"] != user["id"]: self.db.tasklog.add(task["id"], False, msg="no permission error, task disabled.") self.db.task.mod(task["id"], next=None, disabled=1) raise gen.Return(False) start = time.time() try: fetch_tpl = self.db.user.decrypt(0 if not tpl["userid"] else task["userid"], tpl["tpl"]) env = dict(variables=self.db.user.decrypt(task["userid"], task["init_env"]), session=[]) new_env = yield self.fetcher.do_fetch(fetch_tpl, env) variables = self.db.user.encrypt(task["userid"], new_env["variables"]) session = self.db.user.encrypt( task["userid"], new_env["session"] if isinstance(new_env["session"], basestring) else new_env["session"].to_json(), ) # todo next not mid night next = time.time() + (tpl["interval"] if tpl["interval"] else 24 * 60 * 60) if tpl["interval"] is None: next = self.fix_next_time(next) # success feedback self.db.tasklog.add(task["id"], success=True, msg=new_env["variables"].get("__log__")) self.db.task.mod( task["id"], last_success=time.time(), last_failed_count=0, success_count=task["success_count"] + 1, env=variables, session=session, mtime=time.time(), next=next, ) self.db.tpl.incr_success(tpl["id"]) logger.info("taskid:%d tplid:%d successed! %.4fs", task["id"], task["tplid"], time.time() - start) except Exception as e: # failed feedback next_time_delta = self.failed_count_to_time(task["last_failed_count"], tpl["interval"]) if next_time_delta: disabled = False next = time.time() + next_time_delta else: disabled = True next = None self.db.tasklog.add(task["id"], success=False, msg=unicode(e)) self.db.task.mod( task["id"], last_failed=time.time(), failed_count=task["failed_count"] + 1, last_failed_count=task["last_failed_count"] + 1, disabled=disabled, mtime=time.time(), next=next, ) self.db.tpl.incr_failed(tpl["id"]) if ( task["success_count"] and task["last_failed_count"] and user["email_verified"] and user["email"] and self.is_tommorrow(next) ): try: _ = yield utils.send_mail( to=user["email"], subject=u"%s - 签到失败%s" % (tpl["sitename"], u" 已停止" if disabled else u""), text=u""" 您的 %(sitename)s [ %(siteurl)s ] 签到任务,执行 %(cnt)d次 失败。%(disable)s 下一次重试在一天之后,为防止签到中断,给您发送这份邮件。 访问: http://qiandao.today/task/%(taskid)s/log 查看日志。 """ % dict( sitename=tpl["sitename"] or u"未命名", siteurl=tpl["siteurl"] or u"", cnt=task["last_failed_count"] + 1, disable=u"因连续多次失败,已停止。" if disabled else u"", taskid=task["id"], ), async=True, ) except Exception as e: logging.error("send mail error: %r", e) logger.error("taskid:%d tplid:%d failed! %r %.4fs", task["id"], task["tplid"], e, time.time() - start) raise gen.Return(False) raise gen.Return(True) def task_failed(self, task, user, tpl, e): pass
if env_file: try: env = json.load(open(env_file)) except Exception as e: logging.error(e) usage() if 'variables' not in env or not isinstance(env['variables'], dict)\ or 'session' not in env: env = { 'variables': env, 'session': [], } env['variables'].update(variables) # do fetch ioloop = IOLoop.instance() def ioloop_stop(x): ioloop.stop() fetcher = Fetcher() result = fetcher.do_fetch(tpl, env) ioloop.add_future(result, ioloop_stop) ioloop.start() try: result = result.result() except Exception as e: print 'qiandao failed!', e else: print 'qiandao success!', result.get('variables', {}).get('__log__', '')
class MainWorker(object): def __init__(self): self.running = False if config.db_type == 'sqlite3': import sqlite3_db as db else: import db class DB(object): user = db.UserDB() tpl = db.TPLDB() task = db.TaskDB() tasklog = db.TaskLogDB() site = db.SiteDB() self.db = DB self.fetcher = Fetcher() def __call__(self): if self.running: return self.running = self.run() def done(future): self.running = None success, failed = future.result() if success or failed: logger.info('%d task done. %d success, %d failed' % (success + failed, success, failed)) return self.running.add_done_callback(done) def ClearLog(self, taskid): logDay = int(self.db.site.get(1, fields=('logDay'))['logDay']) for log in self.db.tasklog.list(taskid=taskid, fields=('id', 'ctime')): if (time.time() - log['ctime']) > (logDay * 24 * 60 * 60): self.db.tasklog.delete(log['id']) @gen.coroutine def run(self): running = [] success = 0 failed = 0 try: for task in self.scan(): running.append(self.do(task)) if len(running) > 50: logging.debug('scaned %d task, waiting...', len(running)) result = yield running[:10] for each in result: if each: success += 1 else: failed += 1 running = running[10:] logging.debug('scaned %d task, waiting...', len(running)) result = yield running for each in result: if each: success += 1 else: failed += 1 except Exception as e: logging.exception(e) raise gen.Return((success, failed)) scan_fields = ( 'id', 'tplid', 'userid', 'init_env', 'env', 'session', 'last_success', 'last_failed', 'success_count', 'failed_count', 'last_failed_count', 'next', 'disabled', ) def scan(self): return self.db.task.scan(fields=self.scan_fields) @staticmethod def failed_count_to_time(last_failed_count, interval=None): if last_failed_count == 0: next = 10 * 60 elif last_failed_count == 1: next = 110 * 60 elif last_failed_count == 2: next = 240 * 60 elif last_failed_count == 3: next = 360 * 60 elif last_failed_count < 8: next = 11 * 60 * 60 else: next = None if interval is None: interval = 24 * 60 * 60 if next and next > interval / 2: next = interval / 2 return next @staticmethod def fix_next_time(next, gmt_offset=-8 * 60): date = datetime.datetime.utcfromtimestamp(next) local_date = date - datetime.timedelta(minutes=gmt_offset) if local_date.hour < 2: next += 2 * 60 * 60 if local_date.hour > 21: next -= 3 * 60 * 60 return next @staticmethod def is_tommorrow(next, gmt_offset=-8 * 60): date = datetime.datetime.utcfromtimestamp(next) now = datetime.datetime.utcnow() local_date = date - datetime.timedelta(minutes=gmt_offset) local_now = now - datetime.timedelta(minutes=gmt_offset) local_tomorrow = local_now + datetime.timedelta(hours=24) if local_date.day == local_tomorrow.day and not now.hour > 22: return True elif local_date.hour > 22: return True else: return False @gen.coroutine def do(self, task): task['note'] = self.db.task.get(task['id'], fields=('note'))['note'] user = self.db.user.get(task['userid'], fields=('id', 'email', 'email_verified', 'nickname')) tpl = self.db.tpl.get(task['tplid'], fields=('id', 'userid', 'sitename', 'siteurl', 'tpl', 'interval', 'last_success')) ontime = self.db.task.get(task['id'], fields=('ontime', 'ontimeflg', 'pushsw', 'newontime')) newontime = json.loads(ontime["newontime"]) pushsw = json.loads(ontime['pushsw']) notice = self.db.user.get(task['userid'], fields=('skey', 'barkurl', 'noticeflg', 'wxpusher')) temp = notice['wxpusher'].split(";") wxpusher_token = temp[0] if (len(temp) >= 2) else "" wxpusher_uid = temp[1] if (len(temp) >= 2) else "" pushno2b = send2phone.send2phone(barkurl=notice['barkurl']) pushno2s = send2phone.send2phone(skey=notice['skey']) pushno2w = send2phone.send2phone(wxpusher_token=wxpusher_token, wxpusher_uid=wxpusher_uid) pusher = {} pusher["mailpushersw"] = False if (notice['noticeflg'] & 0x80) == 0 else True pusher["barksw"] = False if (notice['noticeflg'] & 0x40) == 0 else True pusher["schansw"] = False if (notice['noticeflg'] & 0x20) == 0 else True pusher["wxpushersw"] = False if (notice['noticeflg'] & 0x10) == 0 else True logtime = json.loads( self.db.user.get(task['userid'], fields=('logtime'))['logtime']) if 'ErrTolerateCnt' not in logtime: logtime['ErrTolerateCnt'] = 0 if task['disabled']: self.db.tasklog.add(task['id'], False, msg='task disabled.') self.db.task.mod(task['id'], next=None, disabled=1) raise gen.Return(False) if not user: self.db.tasklog.add(task['id'], False, msg='no such user, disabled.') self.db.task.mod(task['id'], next=None, disabled=1) raise gen.Return(False) if not tpl: self.db.tasklog.add(task['id'], False, msg='tpl missing, task disabled.') self.db.task.mod(task['id'], next=None, disabled=1) raise gen.Return(False) if tpl['userid'] and tpl['userid'] != user['id']: self.db.tasklog.add(task['id'], False, msg='no permission error, task disabled.') self.db.task.mod(task['id'], next=None, disabled=1) raise gen.Return(False) start = time.time() try: fetch_tpl = self.db.user.decrypt( 0 if not tpl['userid'] else task['userid'], tpl['tpl']) env = dict( variables=self.db.user.decrypt(task['userid'], task['init_env']), session=[], ) new_env = yield self.fetcher.do_fetch(fetch_tpl, env) variables = self.db.user.encrypt(task['userid'], new_env['variables']) session = self.db.user.encrypt( task['userid'], new_env['session'].to_json() if hasattr( new_env['session'], 'to_json') else new_env['session']) # todo next not mid night if (newontime['sw']): next = calNextTimestamp(newontime, True) else: next = time.time() + max( (tpl['interval'] if tpl['interval'] else 24 * 60 * 60), 1 * 60) if tpl['interval'] is None: next = self.fix_next_time(next) # success feedback self.db.tasklog.add(task['id'], success=True, msg=new_env['variables'].get('__log__')) self.db.task.mod(task['id'], last_success=time.time(), last_failed_count=0, success_count=task['success_count'] + 1, env=variables, session=session, mtime=time.time(), next=next) self.db.tpl.incr_success(tpl['id']) if (notice['noticeflg'] & 0x2 != 0): t = datetime.datetime.now().strftime('%m-%d %H:%M:%S') title = u"签到任务 {0}-{1} 成功".format(tpl['sitename'], task['note']) logtemp = new_env['variables'].get('__log__') if (notice['noticeflg'] & 0x2 != 0) and (pushsw['pushen']): if (pusher["barksw"]): pushno2b.send2bark(title, u"{0} 运行成功".format(t)) if (pusher["schansw"]): pushno2s.send2s(title, u"{0} 日志:{1}".format(t, logtemp)) if (pusher["wxpushersw"]): pushno2w.send2wxpusher( title + u"{0} 日志:{1}".format(t, logtemp)) if (pusher["mailpushersw"]): if user['email'] and user['email_verified']: self.task_send_mail( title, u"{0} 日志:{1}".format(t, logtemp), user['email']) logger.info('taskid:%d tplid:%d successed! %.4fs', task['id'], task['tplid'], time.time() - start) # delete log self.ClearLog(task['id']) except Exception as e: # failed feedback next_time_delta = self.failed_count_to_time( task['last_failed_count'], tpl['interval']) t = datetime.datetime.now().strftime('%m-%d %H:%M:%S') title = u"签到任务 {0}-{1} 失败".format(tpl['sitename'], task['note']) content = u"日志:{log}".format(log=e) if next_time_delta: next = time.time() + next_time_delta t = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(next)) content = content + u"下次运行时间:{0}".format(t) # 每次都推送通知 if (logtime['ErrTolerateCnt'] <= task['last_failed_count']): if (notice['noticeflg'] & 0x1 == 1) and (pushsw['pushen']): if (pusher["barksw"]): pushno2b.send2bark(title, u"请自行排查") if (pusher["schansw"]): pushno2s.send2s(title, content) if (pusher["wxpushersw"]): pushno2w.send2wxpusher(title + u" " + content) if (pusher["mailpushersw"]): if user['email'] and user['email_verified']: self.task_send_mail(title, content, user['email']) disabled = False else: disabled = True next = None # 任务禁用时发送通知 if (notice['noticeflg'] & 1 == 1): if (pusher["barksw"]): pushno2b.send2bark(title, u"任务已禁用") if (pusher["schansw"]): pushno2s.send2s(title, u"任务已禁用") if (pusher["wxpushersw"]): pushno2w.send2wxpusher(title + u"任务已禁用") if (pusher["mailpushersw"]): if user['email'] and user['email_verified']: self.task_send_mail(title, u"任务已禁用", user['email']) self.db.tasklog.add(task['id'], success=False, msg=unicode(e)) self.db.task.mod(task['id'], last_failed=time.time(), failed_count=task['failed_count'] + 1, last_failed_count=task['last_failed_count'] + 1, disabled=disabled, mtime=time.time(), next=next) self.db.tpl.incr_failed(tpl['id']) logger.error('taskid:%d tplid:%d failed! %r %.4fs', task['id'], task['tplid'], e, time.time() - start) raise gen.Return(False) raise gen.Return(True) def task_send_mail(self, title, content, email): try: utils.send_mail(to=email, subject=u"在网站{0} {1}".format(config.domain, title), text=content, async=True) except Exception as e: logging.error('tasend mail error: %r', e) return
# http://binux.me # Created on 2014-08-08 21:06:02 import json import os import traceback import time import base64 import random from tornado import gen from .base import * from config import proxies from libs.fetcher import Fetcher from libs.utils import find_encoding from urllib.parse import quote fetcher = Fetcher() class SubscribeHandler(BaseHandler): @tornado.web.addslash @tornado.web.authenticated async def get(self, userid): msg = '' user = self.current_user adminflg = False if (user['id'] == int(userid)) and (user['role'] == u'admin'): adminflg = True repos = json.loads(self.db.site.get(1, fields=('repos'))['repos']) try: if proxies: proxy = random.choice(proxies)
class MainWorker(object): def __init__(self): self.running = False if config.db_type == 'sqlite3': import sqlite3_db as db else: import db class DB(object): user = db.UserDB() tpl = db.TPLDB() task = db.TaskDB() tasklog = db.TaskLogDB() self.db = DB self.fetcher = Fetcher() def __call__(self): if self.running: return self.running = self.run() def done(future): self.running = None success, failed = future.result() if success or failed: logger.info('%d task done. %d success, %d failed' % (success + failed, success, failed)) return self.running.add_done_callback(done) @gen.coroutine def run(self): running = [] success = 0 failed = 0 try: for task in self.scan(): running.append(self.do(task)) if len(running) > 50: logging.debug('scaned %d task, waiting...', len(running)) result = yield running[:10] for each in result: if each: success += 1 else: failed += 1 running = running[10:] logging.debug('scaned %d task, waiting...', len(running)) result = yield running for each in result: if each: success += 1 else: failed += 1 except Exception as e: logging.exception(e) raise gen.Return((success, failed)) scan_fields = ( 'id', 'tplid', 'userid', 'init_env', 'env', 'session', 'last_success', 'last_failed', 'success_count', 'failed_count', 'last_failed_count', 'next', 'disabled', ) def scan(self): return self.db.task.scan(fields=self.scan_fields) @staticmethod def failed_count_to_time(last_failed_count, interval=None): if last_failed_count == 0: next = 10 * 60 elif last_failed_count == 1: next = 110 * 60 elif last_failed_count == 2: next = 240 * 60 elif last_failed_count == 3: next = 360 * 60 elif last_failed_count < 8: next = 11 * 60 * 60 else: next = None if interval is None: interval = 24 * 60 * 60 if next and next > interval / 2: next = interval / 2 return next @staticmethod def fix_next_time(next, gmt_offset=-8 * 60): date = datetime.datetime.utcfromtimestamp(next) local_date = date - datetime.timedelta(minutes=gmt_offset) if local_date.hour < 2: next += 2 * 60 * 60 if local_date.hour > 21: next -= 3 * 60 * 60 return next @staticmethod def is_tommorrow(next, gmt_offset=-8 * 60): date = datetime.datetime.utcfromtimestamp(next) now = datetime.datetime.utcnow() local_date = date - datetime.timedelta(minutes=gmt_offset) local_now = now - datetime.timedelta(minutes=gmt_offset) local_tomorrow = local_now + datetime.timedelta(hours=24) if local_date.day == local_tomorrow.day and not now.hour > 22: return True elif local_date.hour > 22: return True else: return False @gen.coroutine def do(self, task): user = self.db.user.get(task['userid'], fields=('id', 'email', 'email_verified', 'nickname')) tpl = self.db.tpl.get(task['tplid'], fields=('id', 'userid', 'sitename', 'siteurl', 'tpl', 'interval', 'last_success')) if task['disabled']: self.db.tasklog.add(task['id'], False, msg='task disabled.') self.db.task.mod(task['id'], next=None, disabled=1) raise gen.Return(False) if not user: self.db.tasklog.add(task['id'], False, msg='no such user, disabled.') self.db.task.mod(task['id'], next=None, disabled=1) raise gen.Return(False) if not tpl: self.db.tasklog.add(task['id'], False, msg='tpl missing, task disabled.') self.db.task.mod(task['id'], next=None, disabled=1) raise gen.Return(False) if tpl['userid'] and tpl['userid'] != user['id']: self.db.tasklog.add(task['id'], False, msg='no permission error, task disabled.') self.db.task.mod(task['id'], next=None, disabled=1) raise gen.Return(False) start = time.time() try: fetch_tpl = self.db.user.decrypt( 0 if not tpl['userid'] else task['userid'], tpl['tpl']) env = dict( variables=self.db.user.decrypt(task['userid'], task['init_env']), session=[], ) new_env = yield self.fetcher.do_fetch(fetch_tpl, env) variables = self.db.user.encrypt(task['userid'], new_env['variables']) session = self.db.user.encrypt( task['userid'], new_env['session'].to_json() if hasattr( new_env['session'], 'to_json') else new_env['session']) # todo next not mid night next = time.time() + max( (tpl['interval'] if tpl['interval'] else 24 * 60 * 60), 30 * 60 ) + random.choice( (-1, 1)) * (random.randint(1, 500) + random.randint(1, 500)) if tpl['interval'] is None: next = self.fix_next_time(next) # success feedback self.db.tasklog.add(task['id'], success=True, msg=new_env['variables'].get('__log__')) self.db.task.mod(task['id'], last_success=time.time(), last_failed_count=0, success_count=task['success_count'] + 1, env=variables, session=session, mtime=time.time(), next=next) self.db.tpl.incr_success(tpl['id']) logger.info('taskid:%d tplid:%d successed! %.4fs', task['id'], task['tplid'], time.time() - start) except Exception as e: # failed feedback next_time_delta = self.failed_count_to_time( task['last_failed_count'], tpl['interval']) if next_time_delta: disabled = False next = time.time() + next_time_delta else: disabled = True next = None self.db.tasklog.add(task['id'], success=False, msg=unicode(e)) self.db.task.mod(task['id'], last_failed=time.time(), failed_count=task['failed_count'] + 1, last_failed_count=task['last_failed_count'] + 1, disabled=disabled, mtime=time.time(), next=next) self.db.tpl.incr_failed(tpl['id']) if task['success_count'] and task['last_failed_count'] and user['email_verified'] and user['email']\ and self.is_tommorrow(next): try: _ = yield utils.send_mail( to=user['email'], subject=u"%s - 签到失败%s" % (tpl['sitename'], u' 已停止' if disabled else u""), text=u""" 您的 %(sitename)s [ %(siteurl)s ] 签到任务,执行 %(cnt)d次 失败。%(disable)s 下一次重试在一天之后,为防止签到中断,给您发送这份邮件。 访问: http://%(domain)s/task/%(taskid)s/log 查看日志。 """ % dict( sitename=tpl['sitename'] or u'未命名', siteurl=tpl['siteurl'] or u'', cnt=task['last_failed_count'] + 1, disable=u"因连续多次失败,已停止。" if disabled else u"", domain=config.domain, taskid=task['id'], ), async=True) except Exception as e: logging.error('send mail error: %r', e) logger.error('taskid:%d tplid:%d failed! %r %.4fs', task['id'], task['tplid'], e, time.time() - start) raise gen.Return(False) raise gen.Return(True) def task_failed(self, task, user, tpl, e): pass
class MainWorker(object): def __init__(self): self.running = False if config.db_type == 'sqlite3': import sqlite3_db as db else: import db class DB(object): user = db.UserDB() tpl = db.TPLDB() task = db.TaskDB() tasklog = db.TaskLogDB() self.db = DB self.fetcher = Fetcher() def __call__(self): if self.running: return self.running = self.run() def done(future): self.running = None success, failed = future.result() if success or failed: logger.info('%d task done. %d success, %d failed' % (success+failed, success, failed)) return self.running.add_done_callback(done) @gen.coroutine def run(self): running = [] success = 0 failed = 0 try: for task in self.scan(): running.append(self.do(task)) if len(running) > 50: logging.debug('scaned %d task, waiting...', len(running)) result = yield running[:10] for each in result: if each: success += 1 else: failed += 1 running = running[10:] logging.debug('scaned %d task, waiting...', len(running)) result = yield running for each in result: if each: success += 1 else: failed += 1 except Exception as e: logging.exception(e) raise gen.Return((success, failed)) scan_fields = ('id', 'tplid', 'userid', 'init_env', 'env', 'session', 'last_success', 'last_failed', 'success_count', 'failed_count', 'last_failed_count', 'next', 'disabled', ) def scan(self): return self.db.task.scan(fields=self.scan_fields) @staticmethod def failed_count_to_time(last_failed_count, interval=None): if last_failed_count == 0: next = 10 * 60 elif last_failed_count == 1: next = 110 * 60 elif last_failed_count == 2: next = 240 * 60 elif last_failed_count == 3: next = 360 * 60 elif last_failed_count < 8: next = 11 * 60 * 60 else: next = None if interval is None: interval = 24 * 60 * 60 if next and next > interval / 2: next = interval / 2 return next @staticmethod def fix_next_time(next, gmt_offset=-8*60): date = datetime.datetime.utcfromtimestamp(next) local_date = date - datetime.timedelta(minutes=gmt_offset) if local_date.hour < 2: next += 2 * 60 * 60 if local_date.hour > 21: next -= 3 * 60 * 60 return next @staticmethod def is_tommorrow(next, gmt_offset=-8*60): date = datetime.datetime.utcfromtimestamp(next) now = datetime.datetime.utcnow() local_date = date - datetime.timedelta(minutes=gmt_offset) local_now = now - datetime.timedelta(minutes=gmt_offset) local_tomorrow = local_now + datetime.timedelta(hours=24) if local_date.day == local_tomorrow.day and not now.hour > 22: return True elif local_date.hour > 22: return True else: return False @gen.coroutine def do(self, task): user = self.db.user.get(task['userid'], fields=('id', 'email', 'email_verified', 'nickname')) tpl = self.db.tpl.get(task['tplid'], fields=('id', 'userid', 'sitename', 'siteurl', 'tpl', 'interval', 'last_success')) if task['disabled']: self.db.tasklog.add(task['id'], False, msg='task disabled.') self.db.task.mod(task['id'], next=None, disabled=1) raise gen.Return(False) if not user: self.db.tasklog.add(task['id'], False, msg='no such user, disabled.') self.db.task.mod(task['id'], next=None, disabled=1) raise gen.Return(False) if not tpl: self.db.tasklog.add(task['id'], False, msg='tpl missing, task disabled.') self.db.task.mod(task['id'], next=None, disabled=1) raise gen.Return(False) if tpl['userid'] and tpl['userid'] != user['id']: self.db.tasklog.add(task['id'], False, msg='no permission error, task disabled.') self.db.task.mod(task['id'], next=None, disabled=1) raise gen.Return(False) start = time.time() try: fetch_tpl = self.db.user.decrypt(0 if not tpl['userid'] else task['userid'], tpl['tpl']) env = dict( variables = self.db.user.decrypt(task['userid'], task['init_env']), session = [], ) new_env = yield self.fetcher.do_fetch(fetch_tpl, env) variables = self.db.user.encrypt(task['userid'], new_env['variables']) session = self.db.user.encrypt(task['userid'], new_env['session'].to_json() if hasattr(new_env['session'], 'to_json') else new_env['session']) # todo next not mid night next = time.time() + max((tpl['interval'] if tpl['interval'] else 24 * 60 * 60), 30*60) if tpl['interval'] is None: next = self.fix_next_time(next) # success feedback self.db.tasklog.add(task['id'], success=True, msg=new_env['variables'].get('__log__')) self.db.task.mod(task['id'], last_success=time.time(), last_failed_count=0, success_count=task['success_count']+1, env=variables, session=session, mtime=time.time(), next=next) self.db.tpl.incr_success(tpl['id']) logger.info('taskid:%d tplid:%d successed! %.4fs', task['id'], task['tplid'], time.time()-start) except Exception as e: # failed feedback next_time_delta = self.failed_count_to_time(task['last_failed_count'], tpl['interval']) if next_time_delta: disabled = False next = time.time() + next_time_delta else: disabled = True next = None self.db.tasklog.add(task['id'], success=False, msg=unicode(e)) self.db.task.mod(task['id'], last_failed=time.time(), failed_count=task['failed_count']+1, last_failed_count=task['last_failed_count']+1, disabled = disabled, mtime = time.time(), next=next) self.db.tpl.incr_failed(tpl['id']) if task['success_count'] and task['last_failed_count'] and user['email_verified'] and user['email']\ and self.is_tommorrow(next): try: _ = yield utils.send_mail(to=user['email'], subject=u"%s - 签到失败%s" % ( tpl['sitename'], u' 已停止' if disabled else u""), text=u""" 您的 %(sitename)s [ %(siteurl)s ] 签到任务,执行 %(cnt)d次 失败。%(disable)s 下一次重试在一天之后,为防止签到中断,给您发送这份邮件。 访问: http://%(domain)s/task/%(taskid)s/log 查看日志。 """ % dict( sitename=tpl['sitename'] or u'未命名', siteurl=tpl['siteurl'] or u'', cnt=task['last_failed_count'] + 1, disable=u"因连续多次失败,已停止。" if disabled else u"", domain=config.domain, taskid=task['id'], ), async=True) except Exception as e: logging.error('send mail error: %r', e) logger.error('taskid:%d tplid:%d failed! %r %.4fs', task['id'], task['tplid'], e, time.time()-start) raise gen.Return(False) raise gen.Return(True) def task_failed(self, task, user, tpl, e): pass