class MainWorker(object): def __init__(self): self.running = False if config.db_type == 'sqlite3': import sqlite3_db as db else: import db class DB(object): user = db.UserDB() tpl = db.TPLDB() task = db.TaskDB() tasklog = db.TaskLogDB() site = db.SiteDB() pubtpl = db.PubTplDB() self.db = DB self.fetcher = Fetcher() def __call__(self): # self.running = tornado.ioloop.IOLoop.current().spawn_callback(self.run) # if self.running: # success, failed = self.running # if success or failed: # logger.info('%d task done. %d success, %d failed' % (success+failed, success, failed)) if self.running: return self.running = self.run() def done(future): self.running = None success, failed = future.result() if success or failed: logger.info('%d task done. %d success, %d failed' % (success+failed, success, failed)) return self.running.add_done_callback(done) def ClearLog(self, taskid): logDay = int(self.db.site.get(1, fields=('logDay'))['logDay']) for log in self.db.tasklog.list(taskid = taskid, fields=('id', 'ctime')): if (time.time() - log['ctime']) > (logDay * 24 * 60 * 60): self.db.tasklog.delete(log['id']) async def push_batch(self): try: userlist = await asyncio.wait_for(asyncio.get_event_loop().run_in_executor(None, functools.partial(self.db.user.list,fields=('id', 'email', 'status', 'push_batch'))),timeout=3.0) pushtool = pusher() logging.debug('scaned push_batch task, waiting...') if userlist: for user in userlist: userid = user['id'] push_batch = json.loads(user['push_batch']) if user['status'] == "Enable" and push_batch["sw"] and isinstance(push_batch['time'],(float,int)) and time.time() >= push_batch['time']: title = u"定期签到日志推送" delta = push_batch.get("delta", 86400) logtemp = "{}".format(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(push_batch['time']))) tmpdict = {} tmp = "" numlog = 0 task_list = await asyncio.wait_for(asyncio.get_event_loop().run_in_executor(None, functools.partial(self.db.task.list,userid=userid, fields=('id', 'tplid', 'note', 'disabled', 'last_success', 'last_failed', 'pushsw'))),timeout=3.0) for task in task_list: pushsw = json.loads(task['pushsw']) if pushsw["pushen"] and (task["disabled"] == 0 or (task.get("last_success", 0) and task.get("last_success", 0) >= push_batch['time']-delta) or (task.get("last_failed", 0) and task.get("last_failed", 0) >= push_batch['time']-delta)): tmp0 = "" tasklog_list = await asyncio.wait_for(asyncio.get_event_loop().run_in_executor(None, functools.partial(self.db.tasklog.list,taskid=task["id"], fields=('success', 'ctime', 'msg'))),timeout=3.0) for log in tasklog_list: if (push_batch['time'] - delta) < log['ctime'] <= push_batch['time']: tmp0 += "\\r\\n时间: {}\\r\\n日志: {}".format(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(log['ctime'])), log['msg']) numlog += 1 tmplist = tmpdict.get(task['tplid'],[]) if tmp0: tmplist.append("\\r\\n-----任务{0}-{1}-----{2}\\r\\n".format(len(tmplist)+1, task['note'], tmp0)) else: tmplist.append("\\r\\n-----任务{0}-{1}-----\\r\\n记录期间未执行签到,请检查任务! \\r\\n".format(len(tmplist)+1, task['note'])) tmpdict[task['tplid']] = tmplist for tmpkey in tmpdict: tmp = "\\r\\n\\r\\n=====签到: {0}=====".format(self.db.tpl.get(tmpkey, fields=('sitename'))['sitename']) tmp += ''.join(tmpdict[tmpkey]) logtemp += tmp push_batch["time"] = push_batch['time'] + delta self.db.user.mod(userid, push_batch=json.dumps(push_batch)) if tmp and numlog: user_email = user.get('email','Unkown') logger.debug("Start push batch log for {}".format(user_email)) await pushtool.pusher(userid, {"pushen": bool(push_batch.get("sw",False))}, 4080, title, logtemp) logger.info("Success push batch log for {}".format(user_email)) except Exception as e: traceback.print_exc() logger.error('push batch task failed: {}'.format(str(e))) @gen.coroutine def run(self): running = [] success = 0 failed = 0 try: for task in self.scan(): running.append(self.do(task)) if len(running) > 50: logging.debug('scaned %d task, waiting...', len(running)) result = yield running[:10] for each in result: if each: success += 1 else: failed += 1 running = running[10:] logging.debug('scaned %d task, waiting...', len(running)) result = yield running for each in result: if each: success += 1 else: failed += 1 if config.push_batch_sw: yield self.push_batch() except Exception as e: logging.exception(e) return (success, failed) scan_fields = ('id', 'tplid', 'userid', 'init_env', 'env', 'session', 'retry_count', 'retry_interval', 'last_success', 'last_failed', 'success_count', 'failed_count', 'last_failed_count', 'next', 'disabled', ) def scan(self): return self.db.task.scan(fields=self.scan_fields) @staticmethod def failed_count_to_time(last_failed_count, retry_count=8, retry_interval=None, interval=None): next = None if last_failed_count < retry_count or retry_count == -1: if retry_interval: next = retry_interval else: if last_failed_count == 0: next = 10 * 60 elif last_failed_count == 1: next = 110 * 60 elif last_failed_count == 2: next = 4 * 60 * 60 elif last_failed_count == 3: next = 6 * 60 * 60 elif last_failed_count < retry_count or retry_count == -1: next = 11 * 60 * 60 else: next = None elif retry_count == 0: next = None if interval is None: interval = 24 * 60 * 60 if next: next = min(next, interval / 2) return next @staticmethod def fix_next_time(next, gmt_offset=-8*60): date = datetime.datetime.utcfromtimestamp(next) local_date = date - datetime.timedelta(minutes=gmt_offset) if local_date.hour < 2: next += 2 * 60 * 60 if local_date.hour > 21: next -= 3 * 60 * 60 return next @staticmethod def is_tommorrow(next, gmt_offset=-8*60): date = datetime.datetime.utcfromtimestamp(next) now = datetime.datetime.utcnow() local_date = date - datetime.timedelta(minutes=gmt_offset) local_now = now - datetime.timedelta(minutes=gmt_offset) local_tomorrow = local_now + datetime.timedelta(hours=24) if local_date.day == local_tomorrow.day and not now.hour > 22: return True elif local_date.hour > 22: return True else: return False async def do(self, task): task['note'] = self.db.task.get(task['id'], fields=('note'))['note'] user = self.db.user.get(task['userid'], fields=('id', 'email', 'email_verified', 'nickname', 'logtime')) tpl = self.db.tpl.get(task['tplid'], fields=('id', 'userid', 'sitename', 'siteurl', 'tpl', 'interval', 'last_success')) ontime = self.db.task.get(task['id'], fields=('ontime', 'ontimeflg', 'pushsw', 'newontime', 'next')) newontime = json.loads(ontime["newontime"]) pushtool = pusher() caltool = cal() logtime = json.loads(user['logtime']) pushsw = json.loads(ontime['pushsw']) if 'ErrTolerateCnt' not in logtime:logtime['ErrTolerateCnt'] = 0 if task['disabled']: self.db.tasklog.add(task['id'], False, msg='task disabled.') self.db.task.mod(task['id'], next=None, disabled=1) return False if not user: self.db.tasklog.add(task['id'], False, msg='no such user, disabled.') self.db.task.mod(task['id'], next=None, disabled=1) return False if not tpl: self.db.tasklog.add(task['id'], False, msg='tpl missing, task disabled.') self.db.task.mod(task['id'], next=None, disabled=1) return False if tpl['userid'] and tpl['userid'] != user['id']: self.db.tasklog.add(task['id'], False, msg='no permission error, task disabled.') self.db.task.mod(task['id'], next=None, disabled=1) return False start = time.time() try: fetch_tpl = self.db.user.decrypt(0 if not tpl['userid'] else task['userid'], tpl['tpl']) env = dict( variables = self.db.user.decrypt(task['userid'], task['init_env']), session = [], ) url = utils.parse_url(env['variables'].get('_proxy')) if not url: new_env = await gen.convert_yielded(self.fetcher.do_fetch(fetch_tpl, env)) else: proxy = { 'scheme': url['scheme'], 'host': url['host'], 'port': url['port'], 'username': url['username'], 'password': url['password'] } new_env = await gen.convert_yielded(self.fetcher.do_fetch(fetch_tpl, env, [proxy])) variables = self.db.user.encrypt(task['userid'], new_env['variables']) session = self.db.user.encrypt(task['userid'], new_env['session'].to_json() if hasattr(new_env['session'], 'to_json') else new_env['session']) # todo next not mid night if (newontime['sw']): if ('mode' not in newontime): newontime['mode'] = 'ontime' if (newontime['mode'] == 'ontime'): newontime['date'] = (datetime.datetime.now()+datetime.timedelta(days=1)).strftime("%Y-%m-%d") next = caltool.calNextTs(newontime)['ts'] else: next = time.time() + max((tpl['interval'] if tpl['interval'] else 24 * 60 * 60), 1*60) if tpl['interval'] is None: next = self.fix_next_time(next) # success feedback self.db.tasklog.add(task['id'], success=True, msg=new_env['variables'].get('__log__')) self.db.task.mod(task['id'], last_success=time.time(), last_failed_count=0, success_count=task['success_count']+1, env=variables, session=session, mtime=time.time(), next=next) self.db.tpl.incr_success(tpl['id']) t = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') title = u"签到任务 {0}-{1} 成功".format(tpl['sitename'], task['note']) logtemp = new_env['variables'].get('__log__') logtemp = u"{0} \\r\\n日志:{1}".format(t, logtemp) await pushtool.pusher(user['id'], pushsw, 0x2, title, logtemp) logger.info('taskid:%d tplid:%d successed! %.5fs', task['id'], task['tplid'], time.time()-start) # delete log self.ClearLog(task['id']) except Exception as e: # failed feedback traceback.print_exc() next_time_delta = self.failed_count_to_time(task['last_failed_count'], task['retry_count'], task['retry_interval'], tpl['interval']) t = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') title = u"签到任务 {0}-{1} 失败".format(tpl['sitename'], task['note']) content = u"{0} \\r\\n日志:{1}".format(t, str(e)) disabled = False if next_time_delta: next = time.time() + next_time_delta content = content + u" \\r\\n下次运行时间:{0}".format(time.strftime("%Y-%m-%d %H:%M:%S",time.localtime(next))) if (logtime['ErrTolerateCnt'] <= task['last_failed_count']): await pushtool.pusher(user['id'], pushsw, 0x1, title, content) else: disabled = True next = None content = u" \\r\\n任务已禁用" await pushtool.pusher(user['id'], pushsw, 0x1, title, content) self.db.tasklog.add(task['id'], success=False, msg=str(e)) self.db.task.mod(task['id'], last_failed=time.time(), failed_count=task['failed_count']+1, last_failed_count=task['last_failed_count']+1, disabled = disabled, mtime = time.time(), next=next) self.db.tpl.incr_failed(tpl['id']) logger.error('taskid:%d tplid:%d failed! %r %.4fs', task['id'], task['tplid'], str(e), time.time()-start) return False return True
class MainWorker(object): def __init__(self): self.running = False if config.db_type == 'sqlite3': import sqlite3_db as db else: import db class DB(object): user = db.UserDB() tpl = db.TPLDB() task = db.TaskDB() tasklog = db.TaskLogDB() site = db.SiteDB() pubtpl = db.PubTplDB() self.db = DB self.fetcher = Fetcher() def __call__(self): if self.running: return self.running = self.run() def done(future): self.running = None success, failed = future.result() if success or failed: logger.info('%d task done. %d success, %d failed' % (success + failed, success, failed)) return self.running.add_done_callback(done) def ClearLog(self, taskid): logDay = int(self.db.site.get(1, fields=('logDay'))['logDay']) for log in self.db.tasklog.list(taskid=taskid, fields=('id', 'ctime')): if (time.time() - log['ctime']) > (logDay * 24 * 60 * 60): self.db.tasklog.delete(log['id']) @gen.coroutine def run(self): running = [] success = 0 failed = 0 try: for task in self.scan(): running.append(self.do(task)) if len(running) > 50: logging.debug('scaned %d task, waiting...', len(running)) result = yield running[:10] for each in result: if each: success += 1 else: failed += 1 running = running[10:] logging.debug('scaned %d task, waiting...', len(running)) result = yield running for each in result: if each: success += 1 else: failed += 1 except Exception as e: logging.exception(e) raise gen.Return((success, failed)) scan_fields = ( 'id', 'tplid', 'userid', 'init_env', 'env', 'session', 'last_success', 'last_failed', 'success_count', 'failed_count', 'last_failed_count', 'next', 'disabled', ) def scan(self): return self.db.task.scan(fields=self.scan_fields) @staticmethod def failed_count_to_time(last_failed_count, interval=None): if last_failed_count == 0: next = 10 * 60 elif last_failed_count == 1: next = 110 * 60 elif last_failed_count == 2: next = 240 * 60 elif last_failed_count == 3: next = 360 * 60 elif last_failed_count < 8: next = 11 * 60 * 60 else: next = None if interval is None: interval = 24 * 60 * 60 if next and next > interval / 2: next = interval / 2 return next @staticmethod def fix_next_time(next, gmt_offset=-8 * 60): date = datetime.datetime.utcfromtimestamp(next) local_date = date - datetime.timedelta(minutes=gmt_offset) if local_date.hour < 2: next += 2 * 60 * 60 if local_date.hour > 21: next -= 3 * 60 * 60 return next @staticmethod def is_tommorrow(next, gmt_offset=-8 * 60): date = datetime.datetime.utcfromtimestamp(next) now = datetime.datetime.utcnow() local_date = date - datetime.timedelta(minutes=gmt_offset) local_now = now - datetime.timedelta(minutes=gmt_offset) local_tomorrow = local_now + datetime.timedelta(hours=24) if local_date.day == local_tomorrow.day and not now.hour > 22: return True elif local_date.hour > 22: return True else: return False @gen.coroutine def do(self, task): task['note'] = self.db.task.get(task['id'], fields=('note'))['note'] user = self.db.user.get(task['userid'], fields=('id', 'email', 'email_verified', 'nickname', 'logtime')) tpl = self.db.tpl.get(task['tplid'], fields=('id', 'userid', 'sitename', 'siteurl', 'tpl', 'interval', 'last_success')) ontime = self.db.task.get(task['id'], fields=('ontime', 'ontimeflg', 'pushsw', 'newontime', 'next')) newontime = json.loads(ontime["newontime"]) pushtool = pusher() caltool = cal() logtime = json.loads(user['logtime']) pushsw = json.loads(ontime['pushsw']) if 'ErrTolerateCnt' not in logtime: logtime['ErrTolerateCnt'] = 0 if task['disabled']: self.db.tasklog.add(task['id'], False, msg='task disabled.') self.db.task.mod(task['id'], next=None, disabled=1) raise gen.Return(False) if not user: self.db.tasklog.add(task['id'], False, msg='no such user, disabled.') self.db.task.mod(task['id'], next=None, disabled=1) raise gen.Return(False) if not tpl: self.db.tasklog.add(task['id'], False, msg='tpl missing, task disabled.') self.db.task.mod(task['id'], next=None, disabled=1) raise gen.Return(False) if tpl['userid'] and tpl['userid'] != user['id']: self.db.tasklog.add(task['id'], False, msg='no permission error, task disabled.') self.db.task.mod(task['id'], next=None, disabled=1) raise gen.Return(False) start = time.time() try: fetch_tpl = self.db.user.decrypt( 0 if not tpl['userid'] else task['userid'], tpl['tpl']) env = dict( variables=self.db.user.decrypt(task['userid'], task['init_env']), session=[], ) url = utils.parse_url(env['variables'].get('_proxy')) if not url: new_env = yield self.fetcher.do_fetch(fetch_tpl, env) else: proxy = { 'host': url['host'], 'port': url['port'], } new_env = yield self.fetcher.do_fetch(fetch_tpl, env, [proxy]) variables = self.db.user.encrypt(task['userid'], new_env['variables']) session = self.db.user.encrypt( task['userid'], new_env['session'].to_json() if hasattr( new_env['session'], 'to_json') else new_env['session']) # todo next not mid night if (newontime['sw']): if ('mode' not in newontime): newontime['mode'] = 'ontime' if (newontime['mode'] == 'ontime'): newontime['date'] = ( datetime.datetime.now() + datetime.timedelta(days=1)).strftime("%Y-%m-%d") next = caltool.calNextTs(newontime)['ts'] else: next = time.time() + max( (tpl['interval'] if tpl['interval'] else 24 * 60 * 60), 1 * 60) if tpl['interval'] is None: next = self.fix_next_time(next) # success feedback self.db.tasklog.add(task['id'], success=True, msg=new_env['variables'].get('__log__')) self.db.task.mod(task['id'], last_success=time.time(), last_failed_count=0, success_count=task['success_count'] + 1, env=variables, session=session, mtime=time.time(), next=next) self.db.tpl.incr_success(tpl['id']) t = datetime.datetime.now().strftime('%m-%d %H:%M:%S') title = u"签到任务 {0}-{1} 成功".format(tpl['sitename'], task['note']) logtemp = new_env['variables'].get('__log__') logtemp = u"{0} 日志:{1}".format(t, logtemp) pushtool.pusher(user['id'], pushsw, 0x2, title, logtemp) logger.info('taskid:%d tplid:%d successed! %.4fs', task['id'], task['tplid'], time.time() - start) # delete log self.ClearLog(task['id']) except Exception as e: # failed feedback next_time_delta = self.failed_count_to_time( task['last_failed_count'], tpl['interval']) t = datetime.datetime.now().strftime('%m-%d %H:%M:%S') title = u"签到任务 {0}-{1} 失败".format(tpl['sitename'], task['note']) content = u"日志:{log}".format(log=e) disabled = False if next_time_delta: next = time.time() + next_time_delta content = content + u"下次运行时间:{0}".format( time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(next))) if (logtime['ErrTolerateCnt'] <= task['last_failed_count']): pushtool.pusher(user['id'], pushsw, 0x1, title, content) else: disabled = True next = None content = u"任务已禁用" pushtool.pusher(user['id'], pushsw, 0x1, title, content) self.db.tasklog.add(task['id'], success=False, msg=unicode(e)) self.db.task.mod(task['id'], last_failed=time.time(), failed_count=task['failed_count'] + 1, last_failed_count=task['last_failed_count'] + 1, disabled=disabled, mtime=time.time(), next=next) self.db.tpl.incr_failed(tpl['id']) logger.error('taskid:%d tplid:%d failed! %r %.4fs', task['id'], task['tplid'], e, time.time() - start) raise gen.Return(False) raise gen.Return(True)
if env_file: try: env = json.load(open(env_file)) except Exception as e: logging.error(e) usage() if 'variables' not in env or not isinstance(env['variables'], dict)\ or 'session' not in env: env = { 'variables': env, 'session': [], } env['variables'].update(variables) # do fetch ioloop = IOLoop.instance() def ioloop_stop(x): ioloop.stop() fetcher = Fetcher() result = fetcher.do_fetch(tpl, env) ioloop.add_future(result, ioloop_stop) ioloop.start() try: result = result.result() except Exception as e: print 'qiandao failed!', e else: print 'qiandao success!', result.get('variables', {}).get('__log__', '')
class MainWorker(object): def __init__(self): self.running = False if config.db_type == 'sqlite3': import sqlite3_db as db else: import db class DB(object): user = db.UserDB() tpl = db.TPLDB() task = db.TaskDB() tasklog = db.TaskLogDB() site = db.SiteDB() self.db = DB self.fetcher = Fetcher() def __call__(self): if self.running: return self.running = self.run() def done(future): self.running = None success, failed = future.result() if success or failed: logger.info('%d task done. %d success, %d failed' % (success + failed, success, failed)) return self.running.add_done_callback(done) def ClearLog(self, taskid): logDay = int(self.db.site.get(1, fields=('logDay'))['logDay']) for log in self.db.tasklog.list(taskid=taskid, fields=('id', 'ctime')): if (time.time() - log['ctime']) > (logDay * 24 * 60 * 60): self.db.tasklog.delete(log['id']) @gen.coroutine def run(self): running = [] success = 0 failed = 0 try: for task in self.scan(): running.append(self.do(task)) if len(running) > 50: logging.debug('scaned %d task, waiting...', len(running)) result = yield running[:10] for each in result: if each: success += 1 else: failed += 1 running = running[10:] logging.debug('scaned %d task, waiting...', len(running)) result = yield running for each in result: if each: success += 1 else: failed += 1 except Exception as e: logging.exception(e) raise gen.Return((success, failed)) scan_fields = ( 'id', 'tplid', 'userid', 'init_env', 'env', 'session', 'last_success', 'last_failed', 'success_count', 'failed_count', 'last_failed_count', 'next', 'disabled', ) def scan(self): return self.db.task.scan(fields=self.scan_fields) @staticmethod def failed_count_to_time(last_failed_count, interval=None): if last_failed_count == 0: next = 10 * 60 elif last_failed_count == 1: next = 110 * 60 elif last_failed_count == 2: next = 240 * 60 elif last_failed_count == 3: next = 360 * 60 elif last_failed_count < 8: next = 11 * 60 * 60 else: next = None if interval is None: interval = 24 * 60 * 60 if next and next > interval / 2: next = interval / 2 return next @staticmethod def fix_next_time(next, gmt_offset=-8 * 60): date = datetime.datetime.utcfromtimestamp(next) local_date = date - datetime.timedelta(minutes=gmt_offset) if local_date.hour < 2: next += 2 * 60 * 60 if local_date.hour > 21: next -= 3 * 60 * 60 return next @staticmethod def is_tommorrow(next, gmt_offset=-8 * 60): date = datetime.datetime.utcfromtimestamp(next) now = datetime.datetime.utcnow() local_date = date - datetime.timedelta(minutes=gmt_offset) local_now = now - datetime.timedelta(minutes=gmt_offset) local_tomorrow = local_now + datetime.timedelta(hours=24) if local_date.day == local_tomorrow.day and not now.hour > 22: return True elif local_date.hour > 22: return True else: return False @gen.coroutine def do(self, task): task['note'] = self.db.task.get(task['id'], fields=('note'))['note'] user = self.db.user.get(task['userid'], fields=('id', 'email', 'email_verified', 'nickname')) tpl = self.db.tpl.get(task['tplid'], fields=('id', 'userid', 'sitename', 'siteurl', 'tpl', 'interval', 'last_success')) ontime = self.db.task.get(task['id'], fields=('ontime', 'ontimeflg', 'pushsw', 'newontime')) newontime = json.loads(ontime["newontime"]) pushsw = json.loads(ontime['pushsw']) notice = self.db.user.get(task['userid'], fields=('skey', 'barkurl', 'noticeflg', 'wxpusher')) temp = notice['wxpusher'].split(";") wxpusher_token = temp[0] if (len(temp) >= 2) else "" wxpusher_uid = temp[1] if (len(temp) >= 2) else "" pushno2b = send2phone.send2phone(barkurl=notice['barkurl']) pushno2s = send2phone.send2phone(skey=notice['skey']) pushno2w = send2phone.send2phone(wxpusher_token=wxpusher_token, wxpusher_uid=wxpusher_uid) pusher = {} pusher["mailpushersw"] = False if (notice['noticeflg'] & 0x80) == 0 else True pusher["barksw"] = False if (notice['noticeflg'] & 0x40) == 0 else True pusher["schansw"] = False if (notice['noticeflg'] & 0x20) == 0 else True pusher["wxpushersw"] = False if (notice['noticeflg'] & 0x10) == 0 else True logtime = json.loads( self.db.user.get(task['userid'], fields=('logtime'))['logtime']) if 'ErrTolerateCnt' not in logtime: logtime['ErrTolerateCnt'] = 0 if task['disabled']: self.db.tasklog.add(task['id'], False, msg='task disabled.') self.db.task.mod(task['id'], next=None, disabled=1) raise gen.Return(False) if not user: self.db.tasklog.add(task['id'], False, msg='no such user, disabled.') self.db.task.mod(task['id'], next=None, disabled=1) raise gen.Return(False) if not tpl: self.db.tasklog.add(task['id'], False, msg='tpl missing, task disabled.') self.db.task.mod(task['id'], next=None, disabled=1) raise gen.Return(False) if tpl['userid'] and tpl['userid'] != user['id']: self.db.tasklog.add(task['id'], False, msg='no permission error, task disabled.') self.db.task.mod(task['id'], next=None, disabled=1) raise gen.Return(False) start = time.time() try: fetch_tpl = self.db.user.decrypt( 0 if not tpl['userid'] else task['userid'], tpl['tpl']) env = dict( variables=self.db.user.decrypt(task['userid'], task['init_env']), session=[], ) new_env = yield self.fetcher.do_fetch(fetch_tpl, env) variables = self.db.user.encrypt(task['userid'], new_env['variables']) session = self.db.user.encrypt( task['userid'], new_env['session'].to_json() if hasattr( new_env['session'], 'to_json') else new_env['session']) # todo next not mid night if (newontime['sw']): next = calNextTimestamp(newontime, True) else: next = time.time() + max( (tpl['interval'] if tpl['interval'] else 24 * 60 * 60), 1 * 60) if tpl['interval'] is None: next = self.fix_next_time(next) # success feedback self.db.tasklog.add(task['id'], success=True, msg=new_env['variables'].get('__log__')) self.db.task.mod(task['id'], last_success=time.time(), last_failed_count=0, success_count=task['success_count'] + 1, env=variables, session=session, mtime=time.time(), next=next) self.db.tpl.incr_success(tpl['id']) if (notice['noticeflg'] & 0x2 != 0): t = datetime.datetime.now().strftime('%m-%d %H:%M:%S') title = u"签到任务 {0}-{1} 成功".format(tpl['sitename'], task['note']) logtemp = new_env['variables'].get('__log__') if (notice['noticeflg'] & 0x2 != 0) and (pushsw['pushen']): if (pusher["barksw"]): pushno2b.send2bark(title, u"{0} 运行成功".format(t)) if (pusher["schansw"]): pushno2s.send2s(title, u"{0} 日志:{1}".format(t, logtemp)) if (pusher["wxpushersw"]): pushno2w.send2wxpusher( title + u"{0} 日志:{1}".format(t, logtemp)) if (pusher["mailpushersw"]): if user['email'] and user['email_verified']: self.task_send_mail( title, u"{0} 日志:{1}".format(t, logtemp), user['email']) logger.info('taskid:%d tplid:%d successed! %.4fs', task['id'], task['tplid'], time.time() - start) # delete log self.ClearLog(task['id']) except Exception as e: # failed feedback next_time_delta = self.failed_count_to_time( task['last_failed_count'], tpl['interval']) t = datetime.datetime.now().strftime('%m-%d %H:%M:%S') title = u"签到任务 {0}-{1} 失败".format(tpl['sitename'], task['note']) content = u"日志:{log}".format(log=e) if next_time_delta: next = time.time() + next_time_delta t = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(next)) content = content + u"下次运行时间:{0}".format(t) # 每次都推送通知 if (logtime['ErrTolerateCnt'] <= task['last_failed_count']): if (notice['noticeflg'] & 0x1 == 1) and (pushsw['pushen']): if (pusher["barksw"]): pushno2b.send2bark(title, u"请自行排查") if (pusher["schansw"]): pushno2s.send2s(title, content) if (pusher["wxpushersw"]): pushno2w.send2wxpusher(title + u" " + content) if (pusher["mailpushersw"]): if user['email'] and user['email_verified']: self.task_send_mail(title, content, user['email']) disabled = False else: disabled = True next = None # 任务禁用时发送通知 if (notice['noticeflg'] & 1 == 1): if (pusher["barksw"]): pushno2b.send2bark(title, u"任务已禁用") if (pusher["schansw"]): pushno2s.send2s(title, u"任务已禁用") if (pusher["wxpushersw"]): pushno2w.send2wxpusher(title + u"任务已禁用") if (pusher["mailpushersw"]): if user['email'] and user['email_verified']: self.task_send_mail(title, u"任务已禁用", user['email']) self.db.tasklog.add(task['id'], success=False, msg=unicode(e)) self.db.task.mod(task['id'], last_failed=time.time(), failed_count=task['failed_count'] + 1, last_failed_count=task['last_failed_count'] + 1, disabled=disabled, mtime=time.time(), next=next) self.db.tpl.incr_failed(tpl['id']) logger.error('taskid:%d tplid:%d failed! %r %.4fs', task['id'], task['tplid'], e, time.time() - start) raise gen.Return(False) raise gen.Return(True) def task_send_mail(self, title, content, email): try: utils.send_mail(to=email, subject=u"在网站{0} {1}".format(config.domain, title), text=content, async=True) except Exception as e: logging.error('tasend mail error: %r', e) return
class MainWorker(object): def __init__(self): self.running = False if config.db_type == 'sqlite3': import sqlite3_db as db else: import db class DB(object): user = db.UserDB() tpl = db.TPLDB() task = db.TaskDB() tasklog = db.TaskLogDB() self.db = DB self.fetcher = Fetcher() def __call__(self): if self.running: return self.running = self.run() def done(future): self.running = None success, failed = future.result() if success or failed: logger.info('%d task done. %d success, %d failed' % (success + failed, success, failed)) return self.running.add_done_callback(done) @gen.coroutine def run(self): running = [] success = 0 failed = 0 try: for task in self.scan(): running.append(self.do(task)) if len(running) > 50: logging.debug('scaned %d task, waiting...', len(running)) result = yield running[:10] for each in result: if each: success += 1 else: failed += 1 running = running[10:] logging.debug('scaned %d task, waiting...', len(running)) result = yield running for each in result: if each: success += 1 else: failed += 1 except Exception as e: logging.exception(e) raise gen.Return((success, failed)) scan_fields = ( 'id', 'tplid', 'userid', 'init_env', 'env', 'session', 'last_success', 'last_failed', 'success_count', 'failed_count', 'last_failed_count', 'next', 'disabled', ) def scan(self): return self.db.task.scan(fields=self.scan_fields) @staticmethod def failed_count_to_time(last_failed_count, interval=None): if last_failed_count == 0: next = 10 * 60 elif last_failed_count == 1: next = 110 * 60 elif last_failed_count == 2: next = 240 * 60 elif last_failed_count == 3: next = 360 * 60 elif last_failed_count < 8: next = 11 * 60 * 60 else: next = None if interval is None: interval = 24 * 60 * 60 if next and next > interval / 2: next = interval / 2 return next @staticmethod def fix_next_time(next, gmt_offset=-8 * 60): date = datetime.datetime.utcfromtimestamp(next) local_date = date - datetime.timedelta(minutes=gmt_offset) if local_date.hour < 2: next += 2 * 60 * 60 if local_date.hour > 21: next -= 3 * 60 * 60 return next @staticmethod def is_tommorrow(next, gmt_offset=-8 * 60): date = datetime.datetime.utcfromtimestamp(next) now = datetime.datetime.utcnow() local_date = date - datetime.timedelta(minutes=gmt_offset) local_now = now - datetime.timedelta(minutes=gmt_offset) local_tomorrow = local_now + datetime.timedelta(hours=24) if local_date.day == local_tomorrow.day and not now.hour > 22: return True elif local_date.hour > 22: return True else: return False @gen.coroutine def do(self, task): user = self.db.user.get(task['userid'], fields=('id', 'email', 'email_verified', 'nickname')) tpl = self.db.tpl.get(task['tplid'], fields=('id', 'userid', 'sitename', 'siteurl', 'tpl', 'interval', 'last_success')) if task['disabled']: self.db.tasklog.add(task['id'], False, msg='task disabled.') self.db.task.mod(task['id'], next=None, disabled=1) raise gen.Return(False) if not user: self.db.tasklog.add(task['id'], False, msg='no such user, disabled.') self.db.task.mod(task['id'], next=None, disabled=1) raise gen.Return(False) if not tpl: self.db.tasklog.add(task['id'], False, msg='tpl missing, task disabled.') self.db.task.mod(task['id'], next=None, disabled=1) raise gen.Return(False) if tpl['userid'] and tpl['userid'] != user['id']: self.db.tasklog.add(task['id'], False, msg='no permission error, task disabled.') self.db.task.mod(task['id'], next=None, disabled=1) raise gen.Return(False) start = time.time() try: fetch_tpl = self.db.user.decrypt( 0 if not tpl['userid'] else task['userid'], tpl['tpl']) env = dict( variables=self.db.user.decrypt(task['userid'], task['init_env']), session=[], ) new_env = yield self.fetcher.do_fetch(fetch_tpl, env) variables = self.db.user.encrypt(task['userid'], new_env['variables']) session = self.db.user.encrypt( task['userid'], new_env['session'].to_json() if hasattr( new_env['session'], 'to_json') else new_env['session']) # todo next not mid night next = time.time() + max( (tpl['interval'] if tpl['interval'] else 24 * 60 * 60), 30 * 60 ) + random.choice( (-1, 1)) * (random.randint(1, 500) + random.randint(1, 500)) if tpl['interval'] is None: next = self.fix_next_time(next) # success feedback self.db.tasklog.add(task['id'], success=True, msg=new_env['variables'].get('__log__')) self.db.task.mod(task['id'], last_success=time.time(), last_failed_count=0, success_count=task['success_count'] + 1, env=variables, session=session, mtime=time.time(), next=next) self.db.tpl.incr_success(tpl['id']) logger.info('taskid:%d tplid:%d successed! %.4fs', task['id'], task['tplid'], time.time() - start) except Exception as e: # failed feedback next_time_delta = self.failed_count_to_time( task['last_failed_count'], tpl['interval']) if next_time_delta: disabled = False next = time.time() + next_time_delta else: disabled = True next = None self.db.tasklog.add(task['id'], success=False, msg=unicode(e)) self.db.task.mod(task['id'], last_failed=time.time(), failed_count=task['failed_count'] + 1, last_failed_count=task['last_failed_count'] + 1, disabled=disabled, mtime=time.time(), next=next) self.db.tpl.incr_failed(tpl['id']) if task['success_count'] and task['last_failed_count'] and user['email_verified'] and user['email']\ and self.is_tommorrow(next): try: _ = yield utils.send_mail( to=user['email'], subject=u"%s - 签到失败%s" % (tpl['sitename'], u' 已停止' if disabled else u""), text=u""" 您的 %(sitename)s [ %(siteurl)s ] 签到任务,执行 %(cnt)d次 失败。%(disable)s 下一次重试在一天之后,为防止签到中断,给您发送这份邮件。 访问: http://%(domain)s/task/%(taskid)s/log 查看日志。 """ % dict( sitename=tpl['sitename'] or u'未命名', siteurl=tpl['siteurl'] or u'', cnt=task['last_failed_count'] + 1, disable=u"因连续多次失败,已停止。" if disabled else u"", domain=config.domain, taskid=task['id'], ), async=True) except Exception as e: logging.error('send mail error: %r', e) logger.error('taskid:%d tplid:%d failed! %r %.4fs', task['id'], task['tplid'], e, time.time() - start) raise gen.Return(False) raise gen.Return(True) def task_failed(self, task, user, tpl, e): pass
class MainWorker(object): def __init__(self): self.running = False if config.db_type == 'sqlite3': import sqlite3_db as db else: import db class DB(object): user = db.UserDB() tpl = db.TPLDB() task = db.TaskDB() tasklog = db.TaskLogDB() self.db = DB self.fetcher = Fetcher() def __call__(self): if self.running: return self.running = self.run() def done(future): self.running = None success, failed = future.result() if success or failed: logger.info('%d task done. %d success, %d failed' % (success+failed, success, failed)) return self.running.add_done_callback(done) @gen.coroutine def run(self): running = [] success = 0 failed = 0 try: for task in self.scan(): running.append(self.do(task)) if len(running) > 50: logging.debug('scaned %d task, waiting...', len(running)) result = yield running[:10] for each in result: if each: success += 1 else: failed += 1 running = running[10:] logging.debug('scaned %d task, waiting...', len(running)) result = yield running for each in result: if each: success += 1 else: failed += 1 except Exception as e: logging.exception(e) raise gen.Return((success, failed)) scan_fields = ('id', 'tplid', 'userid', 'init_env', 'env', 'session', 'last_success', 'last_failed', 'success_count', 'failed_count', 'last_failed_count', 'next', 'disabled', ) def scan(self): return self.db.task.scan(fields=self.scan_fields) @staticmethod def failed_count_to_time(last_failed_count, interval=None): if last_failed_count == 0: next = 10 * 60 elif last_failed_count == 1: next = 110 * 60 elif last_failed_count == 2: next = 240 * 60 elif last_failed_count == 3: next = 360 * 60 elif last_failed_count < 8: next = 11 * 60 * 60 else: next = None if interval is None: interval = 24 * 60 * 60 if next and next > interval / 2: next = interval / 2 return next @staticmethod def fix_next_time(next, gmt_offset=-8*60): date = datetime.datetime.utcfromtimestamp(next) local_date = date - datetime.timedelta(minutes=gmt_offset) if local_date.hour < 2: next += 2 * 60 * 60 if local_date.hour > 21: next -= 3 * 60 * 60 return next @staticmethod def is_tommorrow(next, gmt_offset=-8*60): date = datetime.datetime.utcfromtimestamp(next) now = datetime.datetime.utcnow() local_date = date - datetime.timedelta(minutes=gmt_offset) local_now = now - datetime.timedelta(minutes=gmt_offset) local_tomorrow = local_now + datetime.timedelta(hours=24) if local_date.day == local_tomorrow.day and not now.hour > 22: return True elif local_date.hour > 22: return True else: return False @gen.coroutine def do(self, task): user = self.db.user.get(task['userid'], fields=('id', 'email', 'email_verified', 'nickname')) tpl = self.db.tpl.get(task['tplid'], fields=('id', 'userid', 'sitename', 'siteurl', 'tpl', 'interval', 'last_success')) if task['disabled']: self.db.tasklog.add(task['id'], False, msg='task disabled.') self.db.task.mod(task['id'], next=None, disabled=1) raise gen.Return(False) if not user: self.db.tasklog.add(task['id'], False, msg='no such user, disabled.') self.db.task.mod(task['id'], next=None, disabled=1) raise gen.Return(False) if not tpl: self.db.tasklog.add(task['id'], False, msg='tpl missing, task disabled.') self.db.task.mod(task['id'], next=None, disabled=1) raise gen.Return(False) if tpl['userid'] and tpl['userid'] != user['id']: self.db.tasklog.add(task['id'], False, msg='no permission error, task disabled.') self.db.task.mod(task['id'], next=None, disabled=1) raise gen.Return(False) start = time.time() try: fetch_tpl = self.db.user.decrypt(0 if not tpl['userid'] else task['userid'], tpl['tpl']) env = dict( variables = self.db.user.decrypt(task['userid'], task['init_env']), session = [], ) new_env = yield self.fetcher.do_fetch(fetch_tpl, env) variables = self.db.user.encrypt(task['userid'], new_env['variables']) session = self.db.user.encrypt(task['userid'], new_env['session'].to_json() if hasattr(new_env['session'], 'to_json') else new_env['session']) # todo next not mid night next = time.time() + max((tpl['interval'] if tpl['interval'] else 24 * 60 * 60), 30*60) if tpl['interval'] is None: next = self.fix_next_time(next) # success feedback self.db.tasklog.add(task['id'], success=True, msg=new_env['variables'].get('__log__')) self.db.task.mod(task['id'], last_success=time.time(), last_failed_count=0, success_count=task['success_count']+1, env=variables, session=session, mtime=time.time(), next=next) self.db.tpl.incr_success(tpl['id']) logger.info('taskid:%d tplid:%d successed! %.4fs', task['id'], task['tplid'], time.time()-start) except Exception as e: # failed feedback next_time_delta = self.failed_count_to_time(task['last_failed_count'], tpl['interval']) if next_time_delta: disabled = False next = time.time() + next_time_delta else: disabled = True next = None self.db.tasklog.add(task['id'], success=False, msg=unicode(e)) self.db.task.mod(task['id'], last_failed=time.time(), failed_count=task['failed_count']+1, last_failed_count=task['last_failed_count']+1, disabled = disabled, mtime = time.time(), next=next) self.db.tpl.incr_failed(tpl['id']) if task['success_count'] and task['last_failed_count'] and user['email_verified'] and user['email']\ and self.is_tommorrow(next): try: _ = yield utils.send_mail(to=user['email'], subject=u"%s - 签到失败%s" % ( tpl['sitename'], u' 已停止' if disabled else u""), text=u""" 您的 %(sitename)s [ %(siteurl)s ] 签到任务,执行 %(cnt)d次 失败。%(disable)s 下一次重试在一天之后,为防止签到中断,给您发送这份邮件。 访问: http://%(domain)s/task/%(taskid)s/log 查看日志。 """ % dict( sitename=tpl['sitename'] or u'未命名', siteurl=tpl['siteurl'] or u'', cnt=task['last_failed_count'] + 1, disable=u"因连续多次失败,已停止。" if disabled else u"", domain=config.domain, taskid=task['id'], ), async=True) except Exception as e: logging.error('send mail error: %r', e) logger.error('taskid:%d tplid:%d failed! %r %.4fs', task['id'], task['tplid'], e, time.time()-start) raise gen.Return(False) raise gen.Return(True) def task_failed(self, task, user, tpl, e): pass
class MainWorker(object): def __init__(self): self.running = False class DB(object): user = db.UserDB() tpl = db.TPLDB() task = db.TaskDB() tasklog = db.TaskLogDB() self.db = DB self.fetcher = Fetcher() def __call__(self): if self.running: return self.running = self.run() def done(future): self.running = None success, failed = future.result() if success or failed: logger.info("%d task done. %d success, %d failed" % (success + failed, success, failed)) return self.running.add_done_callback(done) @gen.coroutine def run(self): running = [] success = 0 failed = 0 try: for task in self.scan(): running.append(self.do(task)) if len(running) > 50: logging.debug("scaned %d task, waiting...", len(running)) result = yield running[:10] for each in result: if each: success += 1 else: failed += 1 running = running[10:] logging.debug("scaned %d task, waiting...", len(running)) result = yield running for each in result: if each: success += 1 else: failed += 1 except Exception as e: logging.exception(e) raise gen.Return((success, failed)) scan_fields = ( "id", "tplid", "userid", "init_env", "env", "session", "last_success", "last_failed", "success_count", "failed_count", "last_failed_count", "next", "disabled", ) def scan(self): return self.db.task.scan(fields=self.scan_fields) @staticmethod def failed_count_to_time(last_failed_count, interval=None): if last_failed_count == 0: next = 10 * 60 elif last_failed_count == 1: next = 110 * 60 elif last_failed_count == 2: next = 240 * 60 elif last_failed_count == 3: next = 360 * 60 elif last_failed_count < 8: next = 11 * 60 * 60 else: next = None if interval is None: interval = 24 * 60 * 60 if next and next > interval / 2: next = interval / 2 return next @staticmethod def fix_next_time(next, gmt_offset=-8 * 60): date = datetime.datetime.utcfromtimestamp(next) local_date = date - datetime.timedelta(minutes=gmt_offset) if local_date.hour < 2: next += 2 * 60 * 60 if local_date.hour > 21: next -= 3 * 60 * 60 return next @staticmethod def is_tommorrow(next, gmt_offset=-8 * 60): date = datetime.datetime.utcfromtimestamp(next) now = datetime.datetime.utcnow() local_date = date - datetime.timedelta(minutes=gmt_offset) local_now = now - datetime.timedelta(minutes=gmt_offset) local_tomorrow = local_now + datetime.timedelta(hours=24) if local_date.day == local_tomorrow.day and not now.hour > 22: return True elif local_date.hour > 22: return True else: return False @gen.coroutine def do(self, task): user = self.db.user.get(task["userid"], fields=("id", "email", "email_verified", "nickname")) tpl = self.db.tpl.get( task["tplid"], fields=("id", "userid", "sitename", "siteurl", "tpl", "interval", "last_success") ) if task["disabled"]: self.db.tasklog.add(task["id"], False, msg="task disabled.") self.db.task.mod(task["id"], next=None, disabled=1) raise gen.Return(False) if not user: self.db.tasklog.add(task["id"], False, msg="no such user, disabled.") self.db.task.mod(task["id"], next=None, disabled=1) raise gen.Return(False) if not tpl: self.db.tasklog.add(task["id"], False, msg="tpl missing, task disabled.") self.db.task.mod(task["id"], next=None, disabled=1) raise gen.Return(False) if tpl["userid"] and tpl["userid"] != user["id"]: self.db.tasklog.add(task["id"], False, msg="no permission error, task disabled.") self.db.task.mod(task["id"], next=None, disabled=1) raise gen.Return(False) start = time.time() try: fetch_tpl = self.db.user.decrypt(0 if not tpl["userid"] else task["userid"], tpl["tpl"]) env = dict(variables=self.db.user.decrypt(task["userid"], task["init_env"]), session=[]) new_env = yield self.fetcher.do_fetch(fetch_tpl, env) variables = self.db.user.encrypt(task["userid"], new_env["variables"]) session = self.db.user.encrypt( task["userid"], new_env["session"] if isinstance(new_env["session"], basestring) else new_env["session"].to_json(), ) # todo next not mid night next = time.time() + (tpl["interval"] if tpl["interval"] else 24 * 60 * 60) if tpl["interval"] is None: next = self.fix_next_time(next) # success feedback self.db.tasklog.add(task["id"], success=True, msg=new_env["variables"].get("__log__")) self.db.task.mod( task["id"], last_success=time.time(), last_failed_count=0, success_count=task["success_count"] + 1, env=variables, session=session, mtime=time.time(), next=next, ) self.db.tpl.incr_success(tpl["id"]) logger.info("taskid:%d tplid:%d successed! %.4fs", task["id"], task["tplid"], time.time() - start) except Exception as e: # failed feedback next_time_delta = self.failed_count_to_time(task["last_failed_count"], tpl["interval"]) if next_time_delta: disabled = False next = time.time() + next_time_delta else: disabled = True next = None self.db.tasklog.add(task["id"], success=False, msg=unicode(e)) self.db.task.mod( task["id"], last_failed=time.time(), failed_count=task["failed_count"] + 1, last_failed_count=task["last_failed_count"] + 1, disabled=disabled, mtime=time.time(), next=next, ) self.db.tpl.incr_failed(tpl["id"]) if ( task["success_count"] and task["last_failed_count"] and user["email_verified"] and user["email"] and self.is_tommorrow(next) ): try: _ = yield utils.send_mail( to=user["email"], subject=u"%s - 签到失败%s" % (tpl["sitename"], u" 已停止" if disabled else u""), text=u""" 您的 %(sitename)s [ %(siteurl)s ] 签到任务,执行 %(cnt)d次 失败。%(disable)s 下一次重试在一天之后,为防止签到中断,给您发送这份邮件。 访问: http://qiandao.today/task/%(taskid)s/log 查看日志。 """ % dict( sitename=tpl["sitename"] or u"未命名", siteurl=tpl["siteurl"] or u"", cnt=task["last_failed_count"] + 1, disable=u"因连续多次失败,已停止。" if disabled else u"", taskid=task["id"], ), async=True, ) except Exception as e: logging.error("send mail error: %r", e) logger.error("taskid:%d tplid:%d failed! %r %.4fs", task["id"], task["tplid"], e, time.time() - start) raise gen.Return(False) raise gen.Return(True) def task_failed(self, task, user, tpl, e): pass