def __init__(self):
self.running = False
class DB(object):
user = db.UserDB()
tpl = db.TPLDB()
task = db.TaskDB()
tasklog = db.TaskLogDB()
self.db = DB
self.fetcher = Fetcher()
def __init__(self):
if config.db_type == 'sqlite3':
import sqlite3_db as db
else:
import db
class DB(object):
user = db.UserDB()
tpl = db.TPLDB()
task = db.TaskDB()
tasklog = db.TaskLogDB()
site = db.SiteDB()
pubtpl = db.PubTplDB()
self.db = DB
self.fetcher = Fetcher()
def __init__(self):
self.running = False
if config.db_type == 'sqlite3':
import sqlite3_db as db
else:
import db
class DB(object):
user = db.UserDB()
tpl = db.TPLDB()
task = db.TaskDB()
tasklog = db.TaskLogDB()
self.db = DB
self.fetcher = Fetcher()
def __init__(self):
TaskBase.__init__(self)
self.resultCount = 0
self.fetcher = Fetcher(m_settings.mfile_database())
self.beanstalkclient = m_settings.beanstalk_client()
self.logger = getLogger(self.__class__.__name__, console_out=False, level="debug")
self.province = "安徽省"
self.city = "合肥"
def __init__(self):
self.running = False
class DB(object):
user = db.UserDB()
tpl = db.TPLDB()
task = db.TaskDB()
tasklog = db.TaskLogDB()
self.db = DB
self.fetcher = Fetcher()
def __init__(self):
TaskBase.__init__(self)
self.resultCount = 0
self.fetcher = Fetcher(m_settings.mfile_database())
self.beans_client = m_settings.beanstalk_client()
self.logger = getLogger(self.__class__.__name__,
console_out=False,
level="debug")
self.province = "湖南 "
self.city = ""
self.all_ip = staticproxy.get_all_proxie()
if not isinstance(self.all_ip, list) or len(self.all_ip) <= 0:
raise Exception('代理初始化异常。。。')
def __init__(self):
self.running = False
if config.db_type == 'sqlite3':
import sqlite3_db as db
else:
import db
class DB(object):
user = db.UserDB()
tpl = db.TPLDB()
task = db.TaskDB()
tasklog = db.TaskLogDB()
self.db = DB
self.fetcher = Fetcher()
def __init__(self):
settings = dict(
template_path = os.path.join(os.path.dirname(__file__), "tpl"),
static_path = os.path.join(os.path.dirname(__file__), "static"),
debug = config.debug,
gzip = config.gzip,
autoreload = config.autoreload,
cookie_secret = config.cookie_secret,
login_url = '/login',
)
version_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), "version.json")
super(Application, self).__init__(handlers, **settings)
self.jinja_env = jinja2.Environment(
loader=jinja2.FileSystemLoader(settings['template_path']),
extensions=['jinja2.ext.autoescape', 'jinja2.ext.loopcontrols', ],
autoescape=True,
auto_reload=config.autoreload)
if config.db_type == 'sqlite3':
import sqlite3_db as db
else:
import db
class DB(object):
user = db.UserDB()
tpl = db.TPLDB()
task = db.TaskDB()
tasklog = db.TaskLogDB()
push_request = db.PRDB()
redis = db.RedisDB()
site = db.SiteDB()
pubtpl = db.PubTplDB()
self.db = DB
self.fetcher = Fetcher()
with open(version_path, "r", encoding='utf-8') as f:
version_data = json.load(f)
self.jinja_env.globals.update({
'config': config,
'format_date': utils.format_date,
'varbinary2ip': utils.varbinary2ip,
'version': version_data['version']
})
self.jinja_env.filters.update(ui_methods)
class MainWorker(object):
def __init__(self):
self.running = False
if config.db_type == 'sqlite3':
import sqlite3_db as db
else:
import db
class DB(object):
user = db.UserDB()
tpl = db.TPLDB()
task = db.TaskDB()
tasklog = db.TaskLogDB()
site = db.SiteDB()
pubtpl = db.PubTplDB()
self.db = DB
self.fetcher = Fetcher()
def __call__(self):
# self.running = tornado.ioloop.IOLoop.current().spawn_callback(self.run)
# if self.running:
# success, failed = self.running
# if success or failed:
# logger.info('%d task done. %d success, %d failed' % (success+failed, success, failed))
if self.running:
return
self.running = self.run()
def done(future):
self.running = None
success, failed = future.result()
if success or failed:
logger.info('%d task done. %d success, %d failed' % (success+failed, success, failed))
return
self.running.add_done_callback(done)
def ClearLog(self, taskid):
logDay = int(self.db.site.get(1, fields=('logDay'))['logDay'])
for log in self.db.tasklog.list(taskid = taskid, fields=('id', 'ctime')):
if (time.time() - log['ctime']) > (logDay * 24 * 60 * 60):
self.db.tasklog.delete(log['id'])
async def push_batch(self):
try:
userlist = await asyncio.wait_for(asyncio.get_event_loop().run_in_executor(None, functools.partial(self.db.user.list,fields=('id', 'email', 'status', 'push_batch'))),timeout=3.0)
pushtool = pusher()
logging.debug('scaned push_batch task, waiting...')
if userlist:
for user in userlist:
userid = user['id']
push_batch = json.loads(user['push_batch'])
if user['status'] == "Enable" and push_batch["sw"] and isinstance(push_batch['time'],(float,int)) and time.time() >= push_batch['time']:
title = u"定期签到日志推送"
delta = push_batch.get("delta", 86400)
logtemp = "{}".format(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(push_batch['time'])))
tmpdict = {}
tmp = ""
numlog = 0
task_list = await asyncio.wait_for(asyncio.get_event_loop().run_in_executor(None, functools.partial(self.db.task.list,userid=userid, fields=('id', 'tplid', 'note', 'disabled', 'last_success', 'last_failed', 'pushsw'))),timeout=3.0)
for task in task_list:
pushsw = json.loads(task['pushsw'])
if pushsw["pushen"] and (task["disabled"] == 0 or (task.get("last_success", 0) and task.get("last_success", 0) >= push_batch['time']-delta) or (task.get("last_failed", 0) and task.get("last_failed", 0) >= push_batch['time']-delta)):
tmp0 = ""
tasklog_list = await asyncio.wait_for(asyncio.get_event_loop().run_in_executor(None, functools.partial(self.db.tasklog.list,taskid=task["id"], fields=('success', 'ctime', 'msg'))),timeout=3.0)
for log in tasklog_list:
if (push_batch['time'] - delta) < log['ctime'] <= push_batch['time']:
tmp0 += "\\r\\n时间: {}\\r\\n日志: {}".format(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(log['ctime'])), log['msg'])
numlog += 1
tmplist = tmpdict.get(task['tplid'],[])
if tmp0:
tmplist.append("\\r\\n-----任务{0}-{1}-----{2}\\r\\n".format(len(tmplist)+1, task['note'], tmp0))
else:
tmplist.append("\\r\\n-----任务{0}-{1}-----\\r\\n记录期间未执行签到，请检查任务! \\r\\n".format(len(tmplist)+1, task['note']))
tmpdict[task['tplid']] = tmplist
for tmpkey in tmpdict:
tmp = "\\r\\n\\r\\n=====签到: {0}=====".format(self.db.tpl.get(tmpkey, fields=('sitename'))['sitename'])
tmp += ''.join(tmpdict[tmpkey])
logtemp += tmp
push_batch["time"] = push_batch['time'] + delta
self.db.user.mod(userid, push_batch=json.dumps(push_batch))
if tmp and numlog:
user_email = user.get('email','Unkown')
logger.debug("Start push batch log for {}".format(user_email))
await pushtool.pusher(userid, {"pushen": bool(push_batch.get("sw",False))}, 4080, title, logtemp)
logger.info("Success push batch log for {}".format(user_email))
except Exception as e:
traceback.print_exc()
logger.error('push batch task failed: {}'.format(str(e)))
@gen.coroutine
def run(self):
running = []
success = 0
failed = 0
try:
for task in self.scan():
running.append(self.do(task))
if len(running) > 50:
logging.debug('scaned %d task, waiting...', len(running))
result = yield running[:10]
for each in result:
if each:
success += 1
else:
failed += 1
running = running[10:]
logging.debug('scaned %d task, waiting...', len(running))
result = yield running
for each in result:
if each:
success += 1
else:
failed += 1
if config.push_batch_sw:
yield self.push_batch()
except Exception as e:
logging.exception(e)
return (success, failed)
scan_fields = ('id', 'tplid', 'userid', 'init_env', 'env', 'session', 'retry_count', 'retry_interval', 'last_success', 'last_failed', 'success_count', 'failed_count', 'last_failed_count', 'next', 'disabled', )
def scan(self):
return self.db.task.scan(fields=self.scan_fields)
@staticmethod
def failed_count_to_time(last_failed_count, retry_count=8, retry_interval=None, interval=None):
next = None
if last_failed_count < retry_count or retry_count == -1:
if retry_interval:
next = retry_interval
else:
if last_failed_count == 0:
next = 10 * 60
elif last_failed_count == 1:
next = 110 * 60
elif last_failed_count == 2:
next = 4 * 60 * 60
elif last_failed_count == 3:
next = 6 * 60 * 60
elif last_failed_count < retry_count or retry_count == -1:
next = 11 * 60 * 60
else:
next = None
elif retry_count == 0:
next = None
if interval is None:
interval = 24 * 60 * 60
if next:
next = min(next, interval / 2)
return next
@staticmethod
def fix_next_time(next, gmt_offset=-8*60):
date = datetime.datetime.utcfromtimestamp(next)
local_date = date - datetime.timedelta(minutes=gmt_offset)
if local_date.hour < 2:
next += 2 * 60 * 60
if local_date.hour > 21:
next -= 3 * 60 * 60
return next
@staticmethod
def is_tommorrow(next, gmt_offset=-8*60):
date = datetime.datetime.utcfromtimestamp(next)
now = datetime.datetime.utcnow()
local_date = date - datetime.timedelta(minutes=gmt_offset)
local_now = now - datetime.timedelta(minutes=gmt_offset)
local_tomorrow = local_now + datetime.timedelta(hours=24)
if local_date.day == local_tomorrow.day and not now.hour > 22:
return True
elif local_date.hour > 22:
return True
else:
return False
async def do(self, task):
task['note'] = self.db.task.get(task['id'], fields=('note'))['note']
user = self.db.user.get(task['userid'], fields=('id', 'email', 'email_verified', 'nickname', 'logtime'))
tpl = self.db.tpl.get(task['tplid'], fields=('id', 'userid', 'sitename', 'siteurl', 'tpl', 'interval', 'last_success'))
ontime = self.db.task.get(task['id'], fields=('ontime', 'ontimeflg', 'pushsw', 'newontime', 'next'))
newontime = json.loads(ontime["newontime"])
pushtool = pusher()
caltool = cal()
logtime = json.loads(user['logtime'])
pushsw = json.loads(ontime['pushsw'])
if 'ErrTolerateCnt' not in logtime:logtime['ErrTolerateCnt'] = 0
if task['disabled']:
self.db.tasklog.add(task['id'], False, msg='task disabled.')
self.db.task.mod(task['id'], next=None, disabled=1)
return False
if not user:
self.db.tasklog.add(task['id'], False, msg='no such user, disabled.')
self.db.task.mod(task['id'], next=None, disabled=1)
return False
if not tpl:
self.db.tasklog.add(task['id'], False, msg='tpl missing, task disabled.')
self.db.task.mod(task['id'], next=None, disabled=1)
return False
if tpl['userid'] and tpl['userid'] != user['id']:
self.db.tasklog.add(task['id'], False, msg='no permission error, task disabled.')
self.db.task.mod(task['id'], next=None, disabled=1)
return False
start = time.time()
try:
fetch_tpl = self.db.user.decrypt(0 if not tpl['userid'] else task['userid'], tpl['tpl'])
env = dict(
variables = self.db.user.decrypt(task['userid'], task['init_env']),
session = [],
)
url = utils.parse_url(env['variables'].get('_proxy'))
if not url:
new_env = await gen.convert_yielded(self.fetcher.do_fetch(fetch_tpl, env))
else:
proxy = {
'scheme': url['scheme'],
'host': url['host'],
'port': url['port'],
'username': url['username'],
'password': url['password']
}
new_env = await gen.convert_yielded(self.fetcher.do_fetch(fetch_tpl, env, [proxy]))
variables = self.db.user.encrypt(task['userid'], new_env['variables'])
session = self.db.user.encrypt(task['userid'],
new_env['session'].to_json() if hasattr(new_env['session'], 'to_json') else new_env['session'])
# todo next not mid night
if (newontime['sw']):
if ('mode' not in newontime):
newontime['mode'] = 'ontime'
if (newontime['mode'] == 'ontime'):
newontime['date'] = (datetime.datetime.now()+datetime.timedelta(days=1)).strftime("%Y-%m-%d")
next = caltool.calNextTs(newontime)['ts']
else:
next = time.time() + max((tpl['interval'] if tpl['interval'] else 24 * 60 * 60), 1*60)
if tpl['interval'] is None:
next = self.fix_next_time(next)
# success feedback
self.db.tasklog.add(task['id'], success=True, msg=new_env['variables'].get('__log__'))
self.db.task.mod(task['id'],
last_success=time.time(),
last_failed_count=0,
success_count=task['success_count']+1,
env=variables,
session=session,
mtime=time.time(),
next=next)
self.db.tpl.incr_success(tpl['id'])
t = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
title = u"签到任务 {0}-{1} 成功".format(tpl['sitename'], task['note'])
logtemp = new_env['variables'].get('__log__')
logtemp = u"{0} \\r\\n日志：{1}".format(t, logtemp)
await pushtool.pusher(user['id'], pushsw, 0x2, title, logtemp)
logger.info('taskid:%d tplid:%d successed! %.5fs', task['id'], task['tplid'], time.time()-start)
# delete log
self.ClearLog(task['id'])
except Exception as e:
# failed feedback
traceback.print_exc()
next_time_delta = self.failed_count_to_time(task['last_failed_count'], task['retry_count'], task['retry_interval'], tpl['interval'])
t = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
title = u"签到任务 {0}-{1} 失败".format(tpl['sitename'], task['note'])
content = u"{0} \\r\\n日志：{1}".format(t, str(e))
disabled = False
if next_time_delta:
next = time.time() + next_time_delta
content = content + u" \\r\\n下次运行时间：{0}".format(time.strftime("%Y-%m-%d %H:%M:%S",time.localtime(next)))
if (logtime['ErrTolerateCnt'] <= task['last_failed_count']):
await pushtool.pusher(user['id'], pushsw, 0x1, title, content)
else:
disabled = True
next = None
content = u" \\r\\n任务已禁用"
await pushtool.pusher(user['id'], pushsw, 0x1, title, content)
self.db.tasklog.add(task['id'], success=False, msg=str(e))
self.db.task.mod(task['id'],
last_failed=time.time(),
failed_count=task['failed_count']+1,
last_failed_count=task['last_failed_count']+1,
disabled = disabled,
mtime = time.time(),
next=next)
self.db.tpl.incr_failed(tpl['id'])
logger.error('taskid:%d tplid:%d failed! %r %.4fs', task['id'], task['tplid'], str(e), time.time()-start)
return False
return True
class MainWorker(object):
def __init__(self):
self.running = False
if config.db_type == 'sqlite3':
import sqlite3_db as db
else:
import db
class DB(object):
user = db.UserDB()
tpl = db.TPLDB()
task = db.TaskDB()
tasklog = db.TaskLogDB()
site = db.SiteDB()
pubtpl = db.PubTplDB()
self.db = DB
self.fetcher = Fetcher()
def __call__(self):
if self.running:
return
self.running = self.run()
def done(future):
self.running = None
success, failed = future.result()
if success or failed:
logger.info('%d task done. %d success, %d failed' %
(success + failed, success, failed))
return
self.running.add_done_callback(done)
def ClearLog(self, taskid):
logDay = int(self.db.site.get(1, fields=('logDay'))['logDay'])
for log in self.db.tasklog.list(taskid=taskid, fields=('id', 'ctime')):
if (time.time() - log['ctime']) > (logDay * 24 * 60 * 60):
self.db.tasklog.delete(log['id'])
@gen.coroutine
def run(self):
running = []
success = 0
failed = 0
try:
for task in self.scan():
running.append(self.do(task))
if len(running) > 50:
logging.debug('scaned %d task, waiting...', len(running))
result = yield running[:10]
for each in result:
if each:
success += 1
else:
failed += 1
running = running[10:]
logging.debug('scaned %d task, waiting...', len(running))
result = yield running
for each in result:
if each:
success += 1
else:
failed += 1
except Exception as e:
logging.exception(e)
raise gen.Return((success, failed))
scan_fields = (
'id',
'tplid',
'userid',
'init_env',
'env',
'session',
'last_success',
'last_failed',
'success_count',
'failed_count',
'last_failed_count',
'next',
'disabled',
)
def scan(self):
return self.db.task.scan(fields=self.scan_fields)
@staticmethod
def failed_count_to_time(last_failed_count, interval=None):
if last_failed_count == 0:
next = 10 * 60
elif last_failed_count == 1:
next = 110 * 60
elif last_failed_count == 2:
next = 240 * 60
elif last_failed_count == 3:
next = 360 * 60
elif last_failed_count < 8:
next = 11 * 60 * 60
else:
next = None
if interval is None:
interval = 24 * 60 * 60
if next and next > interval / 2:
next = interval / 2
return next
@staticmethod
def fix_next_time(next, gmt_offset=-8 * 60):
date = datetime.datetime.utcfromtimestamp(next)
local_date = date - datetime.timedelta(minutes=gmt_offset)
if local_date.hour < 2:
next += 2 * 60 * 60
if local_date.hour > 21:
next -= 3 * 60 * 60
return next
@staticmethod
def is_tommorrow(next, gmt_offset=-8 * 60):
date = datetime.datetime.utcfromtimestamp(next)
now = datetime.datetime.utcnow()
local_date = date - datetime.timedelta(minutes=gmt_offset)
local_now = now - datetime.timedelta(minutes=gmt_offset)
local_tomorrow = local_now + datetime.timedelta(hours=24)
if local_date.day == local_tomorrow.day and not now.hour > 22:
return True
elif local_date.hour > 22:
return True
else:
return False
@gen.coroutine
def do(self, task):
task['note'] = self.db.task.get(task['id'], fields=('note'))['note']
user = self.db.user.get(task['userid'],
fields=('id', 'email', 'email_verified',
'nickname', 'logtime'))
tpl = self.db.tpl.get(task['tplid'],
fields=('id', 'userid', 'sitename', 'siteurl',
'tpl', 'interval', 'last_success'))
ontime = self.db.task.get(task['id'],
fields=('ontime', 'ontimeflg', 'pushsw',
'newontime', 'next'))
newontime = json.loads(ontime["newontime"])
pushtool = pusher()
caltool = cal()
logtime = json.loads(user['logtime'])
pushsw = json.loads(ontime['pushsw'])
if 'ErrTolerateCnt' not in logtime: logtime['ErrTolerateCnt'] = 0
if task['disabled']:
self.db.tasklog.add(task['id'], False, msg='task disabled.')
self.db.task.mod(task['id'], next=None, disabled=1)
raise gen.Return(False)
if not user:
self.db.tasklog.add(task['id'],
False,
msg='no such user, disabled.')
self.db.task.mod(task['id'], next=None, disabled=1)
raise gen.Return(False)
if not tpl:
self.db.tasklog.add(task['id'],
False,
msg='tpl missing, task disabled.')
self.db.task.mod(task['id'], next=None, disabled=1)
raise gen.Return(False)
if tpl['userid'] and tpl['userid'] != user['id']:
self.db.tasklog.add(task['id'],
False,
msg='no permission error, task disabled.')
self.db.task.mod(task['id'], next=None, disabled=1)
raise gen.Return(False)
start = time.time()
try:
fetch_tpl = self.db.user.decrypt(
0 if not tpl['userid'] else task['userid'], tpl['tpl'])
env = dict(
variables=self.db.user.decrypt(task['userid'],
task['init_env']),
session=[],
)
url = utils.parse_url(env['variables'].get('_proxy'))
if not url:
new_env = yield self.fetcher.do_fetch(fetch_tpl, env)
else:
proxy = {
'host': url['host'],
'port': url['port'],
}
new_env = yield self.fetcher.do_fetch(fetch_tpl, env, [proxy])
variables = self.db.user.encrypt(task['userid'],
new_env['variables'])
session = self.db.user.encrypt(
task['userid'], new_env['session'].to_json() if hasattr(
new_env['session'], 'to_json') else new_env['session'])
# todo next not mid night
if (newontime['sw']):
if ('mode' not in newontime):
newontime['mode'] = 'ontime'
if (newontime['mode'] == 'ontime'):
newontime['date'] = (
datetime.datetime.now() +
datetime.timedelta(days=1)).strftime("%Y-%m-%d")
next = caltool.calNextTs(newontime)['ts']
else:
next = time.time() + max(
(tpl['interval'] if tpl['interval'] else 24 * 60 * 60),
1 * 60)
if tpl['interval'] is None:
next = self.fix_next_time(next)
# success feedback
self.db.tasklog.add(task['id'],
success=True,
msg=new_env['variables'].get('__log__'))
self.db.task.mod(task['id'],
last_success=time.time(),
last_failed_count=0,
success_count=task['success_count'] + 1,
env=variables,
session=session,
mtime=time.time(),
next=next)
self.db.tpl.incr_success(tpl['id'])
t = datetime.datetime.now().strftime('%m-%d %H:%M:%S')
title = u"签到任务 {0}-{1} 成功".format(tpl['sitename'], task['note'])
logtemp = new_env['variables'].get('__log__')
logtemp = u"{0} 日志：{1}".format(t, logtemp)
pushtool.pusher(user['id'], pushsw, 0x2, title, logtemp)
logger.info('taskid:%d tplid:%d successed! %.4fs', task['id'],
task['tplid'],
time.time() - start)
# delete log
self.ClearLog(task['id'])
except Exception as e:
# failed feedback
next_time_delta = self.failed_count_to_time(
task['last_failed_count'], tpl['interval'])
t = datetime.datetime.now().strftime('%m-%d %H:%M:%S')
title = u"签到任务 {0}-{1} 失败".format(tpl['sitename'], task['note'])
content = u"日志：{log}".format(log=e)
disabled = False
if next_time_delta:
next = time.time() + next_time_delta
content = content + u"下次运行时间：{0}".format(
time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(next)))
if (logtime['ErrTolerateCnt'] <= task['last_failed_count']):
pushtool.pusher(user['id'], pushsw, 0x1, title, content)
else:
disabled = True
next = None
content = u"任务已禁用"
pushtool.pusher(user['id'], pushsw, 0x1, title, content)
self.db.tasklog.add(task['id'], success=False, msg=unicode(e))
self.db.task.mod(task['id'],
last_failed=time.time(),
failed_count=task['failed_count'] + 1,
last_failed_count=task['last_failed_count'] + 1,
disabled=disabled,
mtime=time.time(),
next=next)
self.db.tpl.incr_failed(tpl['id'])
logger.error('taskid:%d tplid:%d failed! %r %.4fs', task['id'],
task['tplid'], e,
time.time() - start)
raise gen.Return(False)
raise gen.Return(True)
class pusher(object):
def __init__(self):
if config.db_type == 'sqlite3':
import sqlite3_db as db
else:
import db
class DB(object):
user = db.UserDB()
tpl = db.TPLDB()
task = db.TaskDB()
tasklog = db.TaskLogDB()
site = db.SiteDB()
pubtpl = db.PubTplDB()
self.db = DB
self.fetcher = Fetcher()
async def pusher(self, userid, pushsw, flg, title, content):
notice = self.db.user.get(userid, fields=('skey', 'barkurl', 'noticeflg', 'wxpusher', 'qywx_token', 'tg_token', 'dingding_token', 'diypusher'))
if (notice['noticeflg'] & flg != 0):
user = self.db.user.get(userid, fields=('id', 'email', 'email_verified', 'nickname'))
diypusher = notice['diypusher']
if (diypusher != ''):diypusher = json.loads(diypusher)
self.barklink = notice['barkurl']
pusher = {}
pusher["mailpushersw"] = False if (notice['noticeflg'] & 0x80) == 0 else True
pusher["barksw"] = False if (notice['noticeflg'] & 0x40) == 0 else True
pusher["schansw"] = False if (notice['noticeflg'] & 0x20) == 0 else True
pusher["wxpushersw"] = False if (notice['noticeflg'] & 0x10) == 0 else True
pusher["cuspushersw"] = False if (notice['noticeflg'] & 0x100) == 0 else True
pusher["qywxpushersw"] = False if (notice['noticeflg'] & 0x200) == 0 else True
pusher["tgpushersw"] = False if (notice['noticeflg'] & 0x400) == 0 else True
pusher["dingdingpushersw"] = False if (notice['noticeflg'] & 0x800) == 0 else True
def nonepush(*args):
return
if (pushsw['pushen']):
send2bark = self.send2bark if (pusher["barksw"]) else nonepush
send2s = self.send2s if (pusher["schansw"]) else nonepush
send2wxpusher = self.send2wxpusher if (pusher["wxpushersw"]) else nonepush
sendmail = self.sendmail if (pusher["mailpushersw"]) else nonepush
cus_pusher_send = self.cus_pusher_send if (pusher["cuspushersw"]) else nonepush
qywx_pusher_send = self.qywx_pusher_send if (pusher["qywxpushersw"]) else nonepush
send2tg = self.send2tg if (pusher["tgpushersw"]) else nonepush
send2dingding = self.send2dingding if (pusher["dingdingpushersw"]) else nonepush
if (pushsw['pushen']):
await gen.convert_yielded([send2bark(notice['barkurl'], title, content),
send2s(notice['skey'], title, content),
send2wxpusher( notice['wxpusher'], title+u" "+content),
sendmail( user['email'], title, content),
cus_pusher_send( diypusher, title, content),
qywx_pusher_send( notice['qywx_token'], title, content),
send2tg( notice['tg_token'], title, content),
send2dingding(notice['dingding_token'], title, content)])
async def send2bark(self, barklink, title, content):
r = 'False'
try:
link = barklink
if (link[-1] != '/'): link=link+'/'
content = content.replace('\\r\\n','\n')
d = {"title":title,"body":content}
obj = {'request': {'method': 'POST', 'url': link, 'headers': [{'name' : 'Content-Type', 'value': 'application/json; charset=UTF-8'}], 'cookies': [], 'data':json.dumps(d)}, 'rule': {
'success_asserts': [], 'failed_asserts': [], 'extract_variables': []}, 'env': {'variables': {}, 'session': []}}
_,_,res = await gen.convert_yielded(self.fetcher.build_response(obj = obj))
r = 'True'
except Exception as e:
r = traceback.format_exc()
print(r)
return r
async def send2s(self, skey, title, content):
r = 'False'
if (skey != ""):
try:
link = u"https://sctapi.ftqq.com/{0}.send".format(skey.replace(".send", ""))
content = content.replace('\\r\\n','\n\n')
d = {'text': title, 'desp': content}
obj = {'request': {'method': 'POST', 'url': link, 'headers': [{'name' : 'Content-Type', 'value': 'application/x-www-form-urlencoded; charset=UTF-8'}], 'cookies': [], 'data':utils.urllib.parse.urlencode(d)}, 'rule': {
'success_asserts': [], 'failed_asserts': [], 'extract_variables': []}, 'env': {'variables': {}, 'session': []}}
_,_,res = await gen.convert_yielded(self.fetcher.build_response(obj = obj))
r = 'True'
except Exception as e:
r = traceback.format_exc()
print(r)
return r
async def send2tg(self, tg_token, title, content):
r = 'False'
tmp = tg_token.split(';')
tgToken = ''
tgUserId = ''
if len(tmp) >= 2:
tgToken = tmp[0]
tgUserId = tmp[1]
tgHost = tmp[2] if len(tmp) >= 3 else ''
proxy = utils.parse_url(tmp[3]) if len(tmp) >= 4 else ''
pic = tmp[4] if len(tmp) >= 5 else ''
if tgToken and tgUserId:
try:
token = tgToken
chat_id = tgUserId
#TG_BOT的token
#token = os.environ.get('TG_TOKEN')
#用户的ID
#chat_id = os.environ.get('TG_USERID')
if not tgHost:
link = u'https://api.telegram.org/bot{0}/sendMessage'.format(token)
else:
if tgHost[-1]!='/':
tgHost = tgHost + '/'
if 'http://' in tgHost or 'https://' in tgHost:
link = u'{0}bot{1}/sendMessage'.format(tgHost,token)
else:
link = u'https://{0}bot{1}/sendMessage'.format(tgHost,token)
picurl = config.push_pic if pic == '' else pic
content = content.replace('\\r\\n','</pre>\n<pre>')
d = {'chat_id': str(chat_id), 'text': '<b>' + title + '</b>' + '\n<pre>' + content + '</pre>\n' + '------<a href="' + picurl + '">QianDao提醒</a>------', 'disable_web_page_preview':'false', 'parse_mode': 'HTML'}
obj = {'request': {'method': 'POST', 'url': link, 'headers': [{'name' : 'Content-Type', 'value': 'application/json; charset=UTF-8'}], 'cookies': [], 'data':json.dumps(d)}, 'rule': {
'success_asserts': [], 'failed_asserts': [], 'extract_variables': []}, 'env': {'variables': {}, 'session': []}}
if proxy:
_,_,res = await gen.convert_yielded(self.fetcher.build_response(obj = obj, proxy = proxy))
else:
_,_,res = await gen.convert_yielded(self.fetcher.build_response(obj = obj))
r = 'True'
except Exception as e:
r = traceback.format_exc()
print(r)
return r
async def send2dingding(self, dingding_token, title, content):
r = 'False'
tmp = dingding_token.split(';')
if len(tmp) >= 1:
dingding_token = tmp[0]
pic = tmp[1] if len(tmp) >= 2 else ''
if (dingding_token != ""):
try:
link = u"https://oapi.dingtalk.com/robot/send?access_token={0}".format(dingding_token)
picurl = config.push_pic if pic == '' else pic
content = content.replace('\\r\\n','\n\n > ')
d = {"msgtype":"markdown","markdown":{"title":title,"text":"![QianDao](" + picurl + ")\n " + "#### "+ title + "\n > " +content}}
obj = {'request': {'method': 'POST', 'url': link, 'headers': [{'name' : 'Content-Type', 'value': 'application/json; charset=UTF-8'}], 'cookies': [], 'data':json.dumps(d)}, 'rule': {
'success_asserts': [], 'failed_asserts': [], 'extract_variables': []}, 'env': {'variables': {}, 'session': []}}
_,_,res = await gen.convert_yielded(self.fetcher.build_response(obj = obj))
r = 'True'
except Exception as e:
r = traceback.format_exc()
print(r)
return r
async def send2wxpusher(self, wxpusher, content):
r = 'False'
temp = wxpusher.split(";")
wxpusher_token = temp[0] if (len(temp) >= 2) else ""
wxpusher_uid = temp[1] if (len(temp) >= 2) else ""
if (wxpusher_token != "") and (wxpusher_uid != ""):
try:
link = "http://wxpusher.zjiecode.com/api/send/message"
content = content.replace('\\r\\n','\n')
d = {
"appToken":wxpusher_token,
"content":content,
"contentType":3,
"uids":[
wxpusher_uid
]
}
obj = {'request': {'method': 'POST', 'url': link, 'headers': [{'name' : 'Content-Type', 'value': 'application/json; charset=UTF-8'}], 'cookies': [], 'data':json.dumps(d)}, 'rule': {
'success_asserts': [], 'failed_asserts': [], 'extract_variables': []}, 'env': {'variables': {}, 'session': []}}
_,_,res = await gen.convert_yielded(self.fetcher.build_response(obj = obj))
r = 'True'
except Exception as e:
r = traceback.format_exc()
print(r)
return r
async def cus_pusher_send(self, diypusher, t, log):
r = 'False'
try:
curltmp = diypusher['curl'].format(log=log, t=t)
if (diypusher['headers']):
headerstmp = json.loads(diypusher['headers'].replace('{log}', log).replace("{t}", t))
else:
headerstmp = {}
if (diypusher['mode'] == 'POST'):
postDatatmp = diypusher['postData'].replace('{log}', log).replace("{t}", t).replace("\\r\\n","\r\n" )
if (postDatatmp != ''):
postDatatmp = json.loads(postDatatmp)
if headerstmp:
headerstmp.pop('content-type','')
headerstmp.pop('Content-Type','')
if (diypusher['postMethod'] == 'x-www-form-urlencoded'):
headerstmp['Content-Type'] = "application/x-www-form-urlencoded; charset=UTF-8"
headerstmp = [{'name': name, 'value': headerstmp[name]} for name in headerstmp]
obj = {'request': {'method': 'POST', 'url': curltmp, 'headers': headerstmp, 'cookies': [], 'data':utils.urllib.parse.urlencode(postDatatmp)}, 'rule': {
'success_asserts': [], 'failed_asserts': [], 'extract_variables': []}, 'env': {'variables': {}, 'session': []}}
else:
headerstmp['Content-Type'] = "application/json; charset=UTF-8"
headerstmp = [{'name': name, 'value': headerstmp[name]} for name in headerstmp]
obj = {'request': {'method': 'POST', 'url': curltmp, 'headers': headerstmp, 'cookies': [], 'data':json.dumps(postDatatmp)}, 'rule': {
'success_asserts': [], 'failed_asserts': [], 'extract_variables': []}, 'env': {'variables': {}, 'session': []}}
elif (diypusher['mode'] == 'GET'):
headerstmp = [{'name': name, 'value': headerstmp[name]} for name in headerstmp]
obj = {'request': {'method': 'GET', 'url': curltmp, 'headers': headerstmp, 'cookies': []}, 'rule': {
'success_asserts': [], 'failed_asserts': [], 'extract_variables': []}, 'env': {'variables': {}, 'session': []}}
else:
raise Exception(u'模式未选择')
_,_,res = await gen.convert_yielded(self.fetcher.build_response(obj = obj))
if (res.code == 200):
r = "True"
except Exception as e:
r = traceback.format_exc()
return r
async def qywx_pusher_send(self, qywx_token, t, log):
r = 'False'
try:
qywx = {}
tmp = qywx_token.split(';')
if len(tmp) >= 3:
qywx[u'企业ID'] = tmp[0]
qywx[u'应用ID'] = tmp[1]
qywx[u'应用密钥'] = tmp[2]
qywx[u'图片'] = tmp[3] if len(tmp) >= 4 else ''
else:
raise Exception(u'企业微信token错误')
access_url = 'https://qyapi.weixin.qq.com/cgi-bin/gettoken?corpid={id}&corpsecret={secret}'.format(id=qywx[u'企业ID'], secret=qywx[u'应用密钥'])
obj = {'request': {'method': 'GET', 'url': access_url, 'headers': [], 'cookies': []}, 'rule': {
'success_asserts': [], 'failed_asserts': [], 'extract_variables': []}, 'env': {'variables': {}, 'session': []}}
_,_,res = await gen.convert_yielded(self.fetcher.build_response(obj = obj))
get_access_token_res = json.loads(res.body)
if (get_access_token_res['access_token'] != '' and get_access_token_res['errmsg'] == 'ok'):
msgUrl = 'https://qyapi.weixin.qq.com/cgi-bin/message/send?access_token={0}'.format(get_access_token_res['access_token'])
postData = {"touser" : "@all",
"msgtype" : "news",
"agentid" : qywx[u'应用ID'],
"news" : {
"articles" : [
{
"title" : t,
"description" : log.replace("\\r\\n","\n" ),
"url" : "",
"picurl" : config.push_pic if qywx[u'图片'] == '' else qywx[u'图片']
}
]
}
}
obj = {'request': {'method': 'POST', 'url': msgUrl, 'headers': [{'name' : 'Content-Type', 'value': 'application/json; charset=UTF-8'}], 'cookies': [], 'data':json.dumps(postData)}, 'rule': {
'success_asserts': [], 'failed_asserts': [], 'extract_variables': []}, 'env': {'variables': {}, 'session': []}}
_,_,msg_res = await gen.convert_yielded(self.fetcher.build_response(obj = obj))
tmp = json.loads(msg_res.body)
if (tmp['errmsg'] == 'ok' and tmp['errcode'] == 0):
r = 'True'
except Exception as e:
r = traceback.format_exc()
print(r)
return r
async def sendmail(self, email, title, content):
user = self.db.user.get(email=email, fields=('id', 'email', 'email_verified', 'nickname'))
if user['email'] and user['email_verified']:
try:
content = content.replace('\\r\\n','\n')
await gen.convert_yielded(utils.send_mail(to = email,
subject = u"在网站{0} {1}".format(config.domain, title),
text = content,
shark=True))
except Exception as e:
logging.error('tasend mail error: %r', e)
class MainWorker(object):
def __init__(self):
self.running = False
class DB(object):
user = db.UserDB()
tpl = db.TPLDB()
task = db.TaskDB()
tasklog = db.TaskLogDB()
self.db = DB
self.fetcher = Fetcher()
def __call__(self):
if self.running:
return
self.running = self.run()
def done(future):
self.running = None
success, failed = future.result()
if success or failed:
logger.info("%d task done. %d success, %d failed" % (success + failed, success, failed))
return
self.running.add_done_callback(done)
@gen.coroutine
def run(self):
running = []
success = 0
failed = 0
try:
for task in self.scan():
running.append(self.do(task))
if len(running) > 50:
logging.debug("scaned %d task, waiting...", len(running))
result = yield running[:10]
for each in result:
if each:
success += 1
else:
failed += 1
running = running[10:]
logging.debug("scaned %d task, waiting...", len(running))
result = yield running
for each in result:
if each:
success += 1
else:
failed += 1
except Exception as e:
logging.exception(e)
raise gen.Return((success, failed))
scan_fields = (
"id",
"tplid",
"userid",
"init_env",
"env",
"session",
"last_success",
"last_failed",
"success_count",
"failed_count",
"last_failed_count",
"next",
"disabled",
)
def scan(self):
return self.db.task.scan(fields=self.scan_fields)
@staticmethod
def failed_count_to_time(last_failed_count, interval=None):
if last_failed_count == 0:
next = 10 * 60
elif last_failed_count == 1:
next = 110 * 60
elif last_failed_count == 2:
next = 240 * 60
elif last_failed_count == 3:
next = 360 * 60
elif last_failed_count < 8:
next = 11 * 60 * 60
else:
next = None
if interval is None:
interval = 24 * 60 * 60
if next and next > interval / 2:
next = interval / 2
return next
@staticmethod
def fix_next_time(next, gmt_offset=-8 * 60):
date = datetime.datetime.utcfromtimestamp(next)
local_date = date - datetime.timedelta(minutes=gmt_offset)
if local_date.hour < 2:
next += 2 * 60 * 60
if local_date.hour > 21:
next -= 3 * 60 * 60
return next
@staticmethod
def is_tommorrow(next, gmt_offset=-8 * 60):
date = datetime.datetime.utcfromtimestamp(next)
now = datetime.datetime.utcnow()
local_date = date - datetime.timedelta(minutes=gmt_offset)
local_now = now - datetime.timedelta(minutes=gmt_offset)
local_tomorrow = local_now + datetime.timedelta(hours=24)
if local_date.day == local_tomorrow.day and not now.hour > 22:
return True
elif local_date.hour > 22:
return True
else:
return False
@gen.coroutine
def do(self, task):
user = self.db.user.get(task["userid"], fields=("id", "email", "email_verified", "nickname"))
tpl = self.db.tpl.get(
task["tplid"], fields=("id", "userid", "sitename", "siteurl", "tpl", "interval", "last_success")
)
if task["disabled"]:
self.db.tasklog.add(task["id"], False, msg="task disabled.")
self.db.task.mod(task["id"], next=None, disabled=1)
raise gen.Return(False)
if not user:
self.db.tasklog.add(task["id"], False, msg="no such user, disabled.")
self.db.task.mod(task["id"], next=None, disabled=1)
raise gen.Return(False)
if not tpl:
self.db.tasklog.add(task["id"], False, msg="tpl missing, task disabled.")
self.db.task.mod(task["id"], next=None, disabled=1)
raise gen.Return(False)
if tpl["userid"] and tpl["userid"] != user["id"]:
self.db.tasklog.add(task["id"], False, msg="no permission error, task disabled.")
self.db.task.mod(task["id"], next=None, disabled=1)
raise gen.Return(False)
start = time.time()
try:
fetch_tpl = self.db.user.decrypt(0 if not tpl["userid"] else task["userid"], tpl["tpl"])
env = dict(variables=self.db.user.decrypt(task["userid"], task["init_env"]), session=[])
new_env = yield self.fetcher.do_fetch(fetch_tpl, env)
variables = self.db.user.encrypt(task["userid"], new_env["variables"])
session = self.db.user.encrypt(
task["userid"],
new_env["session"] if isinstance(new_env["session"], basestring) else new_env["session"].to_json(),
)
# todo next not mid night
next = time.time() + (tpl["interval"] if tpl["interval"] else 24 * 60 * 60)
if tpl["interval"] is None:
next = self.fix_next_time(next)
# success feedback
self.db.tasklog.add(task["id"], success=True, msg=new_env["variables"].get("__log__"))
self.db.task.mod(
task["id"],
last_success=time.time(),
last_failed_count=0,
success_count=task["success_count"] + 1,
env=variables,
session=session,
mtime=time.time(),
next=next,
)
self.db.tpl.incr_success(tpl["id"])
logger.info("taskid:%d tplid:%d successed! %.4fs", task["id"], task["tplid"], time.time() - start)
except Exception as e:
# failed feedback
next_time_delta = self.failed_count_to_time(task["last_failed_count"], tpl["interval"])
if next_time_delta:
disabled = False
next = time.time() + next_time_delta
else:
disabled = True
next = None
self.db.tasklog.add(task["id"], success=False, msg=unicode(e))
self.db.task.mod(
task["id"],
last_failed=time.time(),
failed_count=task["failed_count"] + 1,
last_failed_count=task["last_failed_count"] + 1,
disabled=disabled,
mtime=time.time(),
next=next,
)
self.db.tpl.incr_failed(tpl["id"])
if (
task["success_count"]
and task["last_failed_count"]
and user["email_verified"]
and user["email"]
and self.is_tommorrow(next)
):
try:
_ = yield utils.send_mail(
to=user["email"],
subject=u"%s - 签到失败%s" % (tpl["sitename"], u" 已停止" if disabled else u""),
text=u"""
您的 %(sitename)s [ %(siteurl)s ] 签到任务，执行 %(cnt)d次 失败。%(disable)s
下一次重试在一天之后，为防止签到中断，给您发送这份邮件。
访问： http://qiandao.today/task/%(taskid)s/log 查看日志。
"""
% dict(
sitename=tpl["sitename"] or u"未命名",
siteurl=tpl["siteurl"] or u"",
cnt=task["last_failed_count"] + 1,
disable=u"因连续多次失败，已停止。" if disabled else u"",
taskid=task["id"],
),
async=True,
)
except Exception as e:
logging.error("send mail error: %r", e)
logger.error("taskid:%d tplid:%d failed! %r %.4fs", task["id"], task["tplid"], e, time.time() - start)
raise gen.Return(False)
raise gen.Return(True)
def task_failed(self, task, user, tpl, e):
pass
if env_file:
try:
env = json.load(open(env_file))
except Exception as e:
logging.error(e)
usage()
if 'variables' not in env or not isinstance(env['variables'], dict)\
or 'session' not in env:
env = {
'variables': env,
'session': [],
}
env['variables'].update(variables)
# do fetch
ioloop = IOLoop.instance()
def ioloop_stop(x):
ioloop.stop()
fetcher = Fetcher()
result = fetcher.do_fetch(tpl, env)
ioloop.add_future(result, ioloop_stop)
ioloop.start()
try:
result = result.result()
except Exception as e:
print 'qiandao failed!', e
else:
print 'qiandao success!', result.get('variables', {}).get('__log__', '')
class MainWorker(object):
def __init__(self):
self.running = False
if config.db_type == 'sqlite3':
import sqlite3_db as db
else:
import db
class DB(object):
user = db.UserDB()
tpl = db.TPLDB()
task = db.TaskDB()
tasklog = db.TaskLogDB()
site = db.SiteDB()
self.db = DB
self.fetcher = Fetcher()
def __call__(self):
if self.running:
return
self.running = self.run()
def done(future):
self.running = None
success, failed = future.result()
if success or failed:
logger.info('%d task done. %d success, %d failed' %
(success + failed, success, failed))
return
self.running.add_done_callback(done)
def ClearLog(self, taskid):
logDay = int(self.db.site.get(1, fields=('logDay'))['logDay'])
for log in self.db.tasklog.list(taskid=taskid, fields=('id', 'ctime')):
if (time.time() - log['ctime']) > (logDay * 24 * 60 * 60):
self.db.tasklog.delete(log['id'])
@gen.coroutine
def run(self):
running = []
success = 0
failed = 0
try:
for task in self.scan():
running.append(self.do(task))
if len(running) > 50:
logging.debug('scaned %d task, waiting...', len(running))
result = yield running[:10]
for each in result:
if each:
success += 1
else:
failed += 1
running = running[10:]
logging.debug('scaned %d task, waiting...', len(running))
result = yield running
for each in result:
if each:
success += 1
else:
failed += 1
except Exception as e:
logging.exception(e)
raise gen.Return((success, failed))
scan_fields = (
'id',
'tplid',
'userid',
'init_env',
'env',
'session',
'last_success',
'last_failed',
'success_count',
'failed_count',
'last_failed_count',
'next',
'disabled',
)
def scan(self):
return self.db.task.scan(fields=self.scan_fields)
@staticmethod
def failed_count_to_time(last_failed_count, interval=None):
if last_failed_count == 0:
next = 10 * 60
elif last_failed_count == 1:
next = 110 * 60
elif last_failed_count == 2:
next = 240 * 60
elif last_failed_count == 3:
next = 360 * 60
elif last_failed_count < 8:
next = 11 * 60 * 60
else:
next = None
if interval is None:
interval = 24 * 60 * 60
if next and next > interval / 2:
next = interval / 2
return next
@staticmethod
def fix_next_time(next, gmt_offset=-8 * 60):
date = datetime.datetime.utcfromtimestamp(next)
local_date = date - datetime.timedelta(minutes=gmt_offset)
if local_date.hour < 2:
next += 2 * 60 * 60
if local_date.hour > 21:
next -= 3 * 60 * 60
return next
@staticmethod
def is_tommorrow(next, gmt_offset=-8 * 60):
date = datetime.datetime.utcfromtimestamp(next)
now = datetime.datetime.utcnow()
local_date = date - datetime.timedelta(minutes=gmt_offset)
local_now = now - datetime.timedelta(minutes=gmt_offset)
local_tomorrow = local_now + datetime.timedelta(hours=24)
if local_date.day == local_tomorrow.day and not now.hour > 22:
return True
elif local_date.hour > 22:
return True
else:
return False
@gen.coroutine
def do(self, task):
task['note'] = self.db.task.get(task['id'], fields=('note'))['note']
user = self.db.user.get(task['userid'],
fields=('id', 'email', 'email_verified',
'nickname'))
tpl = self.db.tpl.get(task['tplid'],
fields=('id', 'userid', 'sitename', 'siteurl',
'tpl', 'interval', 'last_success'))
ontime = self.db.task.get(task['id'],
fields=('ontime', 'ontimeflg', 'pushsw',
'newontime'))
newontime = json.loads(ontime["newontime"])
pushsw = json.loads(ontime['pushsw'])
notice = self.db.user.get(task['userid'],
fields=('skey', 'barkurl', 'noticeflg',
'wxpusher'))
temp = notice['wxpusher'].split(";")
wxpusher_token = temp[0] if (len(temp) >= 2) else ""
wxpusher_uid = temp[1] if (len(temp) >= 2) else ""
pushno2b = send2phone.send2phone(barkurl=notice['barkurl'])
pushno2s = send2phone.send2phone(skey=notice['skey'])
pushno2w = send2phone.send2phone(wxpusher_token=wxpusher_token,
wxpusher_uid=wxpusher_uid)
pusher = {}
pusher["mailpushersw"] = False if (notice['noticeflg']
& 0x80) == 0 else True
pusher["barksw"] = False if (notice['noticeflg'] & 0x40) == 0 else True
pusher["schansw"] = False if (notice['noticeflg']
& 0x20) == 0 else True
pusher["wxpushersw"] = False if (notice['noticeflg']
& 0x10) == 0 else True
logtime = json.loads(
self.db.user.get(task['userid'], fields=('logtime'))['logtime'])
if 'ErrTolerateCnt' not in logtime: logtime['ErrTolerateCnt'] = 0
if task['disabled']:
self.db.tasklog.add(task['id'], False, msg='task disabled.')
self.db.task.mod(task['id'], next=None, disabled=1)
raise gen.Return(False)
if not user:
self.db.tasklog.add(task['id'],
False,
msg='no such user, disabled.')
self.db.task.mod(task['id'], next=None, disabled=1)
raise gen.Return(False)
if not tpl:
self.db.tasklog.add(task['id'],
False,
msg='tpl missing, task disabled.')
self.db.task.mod(task['id'], next=None, disabled=1)
raise gen.Return(False)
if tpl['userid'] and tpl['userid'] != user['id']:
self.db.tasklog.add(task['id'],
False,
msg='no permission error, task disabled.')
self.db.task.mod(task['id'], next=None, disabled=1)
raise gen.Return(False)
start = time.time()
try:
fetch_tpl = self.db.user.decrypt(
0 if not tpl['userid'] else task['userid'], tpl['tpl'])
env = dict(
variables=self.db.user.decrypt(task['userid'],
task['init_env']),
session=[],
)
new_env = yield self.fetcher.do_fetch(fetch_tpl, env)
variables = self.db.user.encrypt(task['userid'],
new_env['variables'])
session = self.db.user.encrypt(
task['userid'], new_env['session'].to_json() if hasattr(
new_env['session'], 'to_json') else new_env['session'])
# todo next not mid night
if (newontime['sw']):
next = calNextTimestamp(newontime, True)
else:
next = time.time() + max(
(tpl['interval'] if tpl['interval'] else 24 * 60 * 60),
1 * 60)
if tpl['interval'] is None:
next = self.fix_next_time(next)
# success feedback
self.db.tasklog.add(task['id'],
success=True,
msg=new_env['variables'].get('__log__'))
self.db.task.mod(task['id'],
last_success=time.time(),
last_failed_count=0,
success_count=task['success_count'] + 1,
env=variables,
session=session,
mtime=time.time(),
next=next)
self.db.tpl.incr_success(tpl['id'])
if (notice['noticeflg'] & 0x2 != 0):
t = datetime.datetime.now().strftime('%m-%d %H:%M:%S')
title = u"签到任务 {0}-{1} 成功".format(tpl['sitename'],
task['note'])
logtemp = new_env['variables'].get('__log__')
if (notice['noticeflg'] & 0x2 != 0) and (pushsw['pushen']):
if (pusher["barksw"]):
pushno2b.send2bark(title, u"{0} 运行成功".format(t))
if (pusher["schansw"]):
pushno2s.send2s(title,
u"{0} 日志：{1}".format(t, logtemp))
if (pusher["wxpushersw"]):
pushno2w.send2wxpusher(
title + u"{0} 日志：{1}".format(t, logtemp))
if (pusher["mailpushersw"]):
if user['email'] and user['email_verified']:
self.task_send_mail(
title, u"{0} 日志：{1}".format(t, logtemp),
user['email'])
logger.info('taskid:%d tplid:%d successed! %.4fs', task['id'],
task['tplid'],
time.time() - start)
# delete log
self.ClearLog(task['id'])
except Exception as e:
# failed feedback
next_time_delta = self.failed_count_to_time(
task['last_failed_count'], tpl['interval'])
t = datetime.datetime.now().strftime('%m-%d %H:%M:%S')
title = u"签到任务 {0}-{1} 失败".format(tpl['sitename'], task['note'])
content = u"日志：{log}".format(log=e)
if next_time_delta:
next = time.time() + next_time_delta
t = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(next))
content = content + u"下次运行时间：{0}".format(t)
# 每次都推送通知
if (logtime['ErrTolerateCnt'] <= task['last_failed_count']):
if (notice['noticeflg'] & 0x1 == 1) and (pushsw['pushen']):
if (pusher["barksw"]):
pushno2b.send2bark(title, u"请自行排查")
if (pusher["schansw"]): pushno2s.send2s(title, content)
if (pusher["wxpushersw"]):
pushno2w.send2wxpusher(title + u" " + content)
if (pusher["mailpushersw"]):
if user['email'] and user['email_verified']:
self.task_send_mail(title, content,
user['email'])
disabled = False
else:
disabled = True
next = None
# 任务禁用时发送通知
if (notice['noticeflg'] & 1 == 1):
if (pusher["barksw"]): pushno2b.send2bark(title, u"任务已禁用")
if (pusher["schansw"]): pushno2s.send2s(title, u"任务已禁用")
if (pusher["wxpushersw"]):
pushno2w.send2wxpusher(title + u"任务已禁用")
if (pusher["mailpushersw"]):
if user['email'] and user['email_verified']:
self.task_send_mail(title, u"任务已禁用", user['email'])
self.db.tasklog.add(task['id'], success=False, msg=unicode(e))
self.db.task.mod(task['id'],
last_failed=time.time(),
failed_count=task['failed_count'] + 1,
last_failed_count=task['last_failed_count'] + 1,
disabled=disabled,
mtime=time.time(),
next=next)
self.db.tpl.incr_failed(tpl['id'])
logger.error('taskid:%d tplid:%d failed! %r %.4fs', task['id'],
task['tplid'], e,
time.time() - start)
raise gen.Return(False)
raise gen.Return(True)
def task_send_mail(self, title, content, email):
try:
utils.send_mail(to=email,
subject=u"在网站{0} {1}".format(config.domain, title),
text=content,
async=True)
except Exception as e:
logging.error('tasend mail error: %r', e)
return
# http://binux.me
# Created on 2014-08-08 21:06:02
import json
import os
import traceback
import time
import base64
import random
from tornado import gen
from .base import *
from config import proxies
from libs.fetcher import Fetcher
from libs.utils import find_encoding
from urllib.parse import quote
fetcher = Fetcher()
class SubscribeHandler(BaseHandler):
@tornado.web.addslash
@tornado.web.authenticated
async def get(self, userid):
msg = ''
user = self.current_user
adminflg = False
if (user['id'] == int(userid)) and (user['role'] == u'admin'):
adminflg = True
repos = json.loads(self.db.site.get(1, fields=('repos'))['repos'])
try:
if proxies:
proxy = random.choice(proxies)
class MainWorker(object):
def __init__(self):
self.running = False
if config.db_type == 'sqlite3':
import sqlite3_db as db
else:
import db
class DB(object):
user = db.UserDB()
tpl = db.TPLDB()
task = db.TaskDB()
tasklog = db.TaskLogDB()
self.db = DB
self.fetcher = Fetcher()
def __call__(self):
if self.running:
return
self.running = self.run()
def done(future):
self.running = None
success, failed = future.result()
if success or failed:
logger.info('%d task done. %d success, %d failed' %
(success + failed, success, failed))
return
self.running.add_done_callback(done)
@gen.coroutine
def run(self):
running = []
success = 0
failed = 0
try:
for task in self.scan():
running.append(self.do(task))
if len(running) > 50:
logging.debug('scaned %d task, waiting...', len(running))
result = yield running[:10]
for each in result:
if each:
success += 1
else:
failed += 1
running = running[10:]
logging.debug('scaned %d task, waiting...', len(running))
result = yield running
for each in result:
if each:
success += 1
else:
failed += 1
except Exception as e:
logging.exception(e)
raise gen.Return((success, failed))
scan_fields = (
'id',
'tplid',
'userid',
'init_env',
'env',
'session',
'last_success',
'last_failed',
'success_count',
'failed_count',
'last_failed_count',
'next',
'disabled',
)
def scan(self):
return self.db.task.scan(fields=self.scan_fields)
@staticmethod
def failed_count_to_time(last_failed_count, interval=None):
if last_failed_count == 0:
next = 10 * 60
elif last_failed_count == 1:
next = 110 * 60
elif last_failed_count == 2:
next = 240 * 60
elif last_failed_count == 3:
next = 360 * 60
elif last_failed_count < 8:
next = 11 * 60 * 60
else:
next = None
if interval is None:
interval = 24 * 60 * 60
if next and next > interval / 2:
next = interval / 2
return next
@staticmethod
def fix_next_time(next, gmt_offset=-8 * 60):
date = datetime.datetime.utcfromtimestamp(next)
local_date = date - datetime.timedelta(minutes=gmt_offset)
if local_date.hour < 2:
next += 2 * 60 * 60
if local_date.hour > 21:
next -= 3 * 60 * 60
return next
@staticmethod
def is_tommorrow(next, gmt_offset=-8 * 60):
date = datetime.datetime.utcfromtimestamp(next)
now = datetime.datetime.utcnow()
local_date = date - datetime.timedelta(minutes=gmt_offset)
local_now = now - datetime.timedelta(minutes=gmt_offset)
local_tomorrow = local_now + datetime.timedelta(hours=24)
if local_date.day == local_tomorrow.day and not now.hour > 22:
return True
elif local_date.hour > 22:
return True
else:
return False
@gen.coroutine
def do(self, task):
user = self.db.user.get(task['userid'],
fields=('id', 'email', 'email_verified',
'nickname'))
tpl = self.db.tpl.get(task['tplid'],
fields=('id', 'userid', 'sitename', 'siteurl',
'tpl', 'interval', 'last_success'))
if task['disabled']:
self.db.tasklog.add(task['id'], False, msg='task disabled.')
self.db.task.mod(task['id'], next=None, disabled=1)
raise gen.Return(False)
if not user:
self.db.tasklog.add(task['id'],
False,
msg='no such user, disabled.')
self.db.task.mod(task['id'], next=None, disabled=1)
raise gen.Return(False)
if not tpl:
self.db.tasklog.add(task['id'],
False,
msg='tpl missing, task disabled.')
self.db.task.mod(task['id'], next=None, disabled=1)
raise gen.Return(False)
if tpl['userid'] and tpl['userid'] != user['id']:
self.db.tasklog.add(task['id'],
False,
msg='no permission error, task disabled.')
self.db.task.mod(task['id'], next=None, disabled=1)
raise gen.Return(False)
start = time.time()
try:
fetch_tpl = self.db.user.decrypt(
0 if not tpl['userid'] else task['userid'], tpl['tpl'])
env = dict(
variables=self.db.user.decrypt(task['userid'],
task['init_env']),
session=[],
)
new_env = yield self.fetcher.do_fetch(fetch_tpl, env)
variables = self.db.user.encrypt(task['userid'],
new_env['variables'])
session = self.db.user.encrypt(
task['userid'], new_env['session'].to_json() if hasattr(
new_env['session'], 'to_json') else new_env['session'])
# todo next not mid night
next = time.time() + max(
(tpl['interval'] if tpl['interval'] else 24 * 60 * 60), 30 * 60
) + random.choice(
(-1, 1)) * (random.randint(1, 500) + random.randint(1, 500))
if tpl['interval'] is None:
next = self.fix_next_time(next)
# success feedback
self.db.tasklog.add(task['id'],
success=True,
msg=new_env['variables'].get('__log__'))
self.db.task.mod(task['id'],
last_success=time.time(),
last_failed_count=0,
success_count=task['success_count'] + 1,
env=variables,
session=session,
mtime=time.time(),
next=next)
self.db.tpl.incr_success(tpl['id'])
logger.info('taskid:%d tplid:%d successed! %.4fs', task['id'],
task['tplid'],
time.time() - start)
except Exception as e:
# failed feedback
next_time_delta = self.failed_count_to_time(
task['last_failed_count'], tpl['interval'])
if next_time_delta:
disabled = False
next = time.time() + next_time_delta
else:
disabled = True
next = None
self.db.tasklog.add(task['id'], success=False, msg=unicode(e))
self.db.task.mod(task['id'],
last_failed=time.time(),
failed_count=task['failed_count'] + 1,
last_failed_count=task['last_failed_count'] + 1,
disabled=disabled,
mtime=time.time(),
next=next)
self.db.tpl.incr_failed(tpl['id'])
if task['success_count'] and task['last_failed_count'] and user['email_verified'] and user['email']\
and self.is_tommorrow(next):
try:
_ = yield utils.send_mail(
to=user['email'],
subject=u"%s - 签到失败%s" %
(tpl['sitename'], u' 已停止' if disabled else u""),
text=u"""
您的 %(sitename)s [ %(siteurl)s ] 签到任务，执行 %(cnt)d次 失败。%(disable)s
下一次重试在一天之后，为防止签到中断，给您发送这份邮件。
访问： http://%(domain)s/task/%(taskid)s/log 查看日志。
""" % dict(
sitename=tpl['sitename'] or u'未命名',
siteurl=tpl['siteurl'] or u'',
cnt=task['last_failed_count'] + 1,
disable=u"因连续多次失败，已停止。" if disabled else u"",
domain=config.domain,
taskid=task['id'],
),
async=True)
except Exception as e:
logging.error('send mail error: %r', e)
logger.error('taskid:%d tplid:%d failed! %r %.4fs', task['id'],
task['tplid'], e,
time.time() - start)
raise gen.Return(False)
raise gen.Return(True)
def task_failed(self, task, user, tpl, e):
pass
class MainWorker(object):
def __init__(self):
self.running = False
if config.db_type == 'sqlite3':
import sqlite3_db as db
else:
import db
class DB(object):
user = db.UserDB()
tpl = db.TPLDB()
task = db.TaskDB()
tasklog = db.TaskLogDB()
self.db = DB
self.fetcher = Fetcher()
def __call__(self):
if self.running:
return
self.running = self.run()
def done(future):
self.running = None
success, failed = future.result()
if success or failed:
logger.info('%d task done. %d success, %d failed' % (success+failed, success, failed))
return
self.running.add_done_callback(done)
@gen.coroutine
def run(self):
running = []
success = 0
failed = 0
try:
for task in self.scan():
running.append(self.do(task))
if len(running) > 50:
logging.debug('scaned %d task, waiting...', len(running))
result = yield running[:10]
for each in result:
if each:
success += 1
else:
failed += 1
running = running[10:]
logging.debug('scaned %d task, waiting...', len(running))
result = yield running
for each in result:
if each:
success += 1
else:
failed += 1
except Exception as e:
logging.exception(e)
raise gen.Return((success, failed))
scan_fields = ('id', 'tplid', 'userid', 'init_env', 'env', 'session', 'last_success', 'last_failed', 'success_count', 'failed_count', 'last_failed_count', 'next', 'disabled', )
def scan(self):
return self.db.task.scan(fields=self.scan_fields)
@staticmethod
def failed_count_to_time(last_failed_count, interval=None):
if last_failed_count == 0:
next = 10 * 60
elif last_failed_count == 1:
next = 110 * 60
elif last_failed_count == 2:
next = 240 * 60
elif last_failed_count == 3:
next = 360 * 60
elif last_failed_count < 8:
next = 11 * 60 * 60
else:
next = None
if interval is None:
interval = 24 * 60 * 60
if next and next > interval / 2:
next = interval / 2
return next
@staticmethod
def fix_next_time(next, gmt_offset=-8*60):
date = datetime.datetime.utcfromtimestamp(next)
local_date = date - datetime.timedelta(minutes=gmt_offset)
if local_date.hour < 2:
next += 2 * 60 * 60
if local_date.hour > 21:
next -= 3 * 60 * 60
return next
@staticmethod
def is_tommorrow(next, gmt_offset=-8*60):
date = datetime.datetime.utcfromtimestamp(next)
now = datetime.datetime.utcnow()
local_date = date - datetime.timedelta(minutes=gmt_offset)
local_now = now - datetime.timedelta(minutes=gmt_offset)
local_tomorrow = local_now + datetime.timedelta(hours=24)
if local_date.day == local_tomorrow.day and not now.hour > 22:
return True
elif local_date.hour > 22:
return True
else:
return False
@gen.coroutine
def do(self, task):
user = self.db.user.get(task['userid'], fields=('id', 'email', 'email_verified', 'nickname'))
tpl = self.db.tpl.get(task['tplid'], fields=('id', 'userid', 'sitename', 'siteurl', 'tpl',
'interval', 'last_success'))
if task['disabled']:
self.db.tasklog.add(task['id'], False, msg='task disabled.')
self.db.task.mod(task['id'], next=None, disabled=1)
raise gen.Return(False)
if not user:
self.db.tasklog.add(task['id'], False, msg='no such user, disabled.')
self.db.task.mod(task['id'], next=None, disabled=1)
raise gen.Return(False)
if not tpl:
self.db.tasklog.add(task['id'], False, msg='tpl missing, task disabled.')
self.db.task.mod(task['id'], next=None, disabled=1)
raise gen.Return(False)
if tpl['userid'] and tpl['userid'] != user['id']:
self.db.tasklog.add(task['id'], False, msg='no permission error, task disabled.')
self.db.task.mod(task['id'], next=None, disabled=1)
raise gen.Return(False)
start = time.time()
try:
fetch_tpl = self.db.user.decrypt(0 if not tpl['userid'] else task['userid'], tpl['tpl'])
env = dict(
variables = self.db.user.decrypt(task['userid'], task['init_env']),
session = [],
)
new_env = yield self.fetcher.do_fetch(fetch_tpl, env)
variables = self.db.user.encrypt(task['userid'], new_env['variables'])
session = self.db.user.encrypt(task['userid'],
new_env['session'].to_json() if hasattr(new_env['session'], 'to_json') else new_env['session'])
# todo next not mid night
next = time.time() + max((tpl['interval'] if tpl['interval'] else 24 * 60 * 60), 30*60)
if tpl['interval'] is None:
next = self.fix_next_time(next)
# success feedback
self.db.tasklog.add(task['id'], success=True, msg=new_env['variables'].get('__log__'))
self.db.task.mod(task['id'],
last_success=time.time(),
last_failed_count=0,
success_count=task['success_count']+1,
env=variables,
session=session,
mtime=time.time(),
next=next)
self.db.tpl.incr_success(tpl['id'])
logger.info('taskid:%d tplid:%d successed! %.4fs', task['id'], task['tplid'], time.time()-start)
except Exception as e:
# failed feedback
next_time_delta = self.failed_count_to_time(task['last_failed_count'], tpl['interval'])
if next_time_delta:
disabled = False
next = time.time() + next_time_delta
else:
disabled = True
next = None
self.db.tasklog.add(task['id'], success=False, msg=unicode(e))
self.db.task.mod(task['id'],
last_failed=time.time(),
failed_count=task['failed_count']+1,
last_failed_count=task['last_failed_count']+1,
disabled = disabled,
mtime = time.time(),
next=next)
self.db.tpl.incr_failed(tpl['id'])
if task['success_count'] and task['last_failed_count'] and user['email_verified'] and user['email']\
and self.is_tommorrow(next):
try:
_ = yield utils.send_mail(to=user['email'], subject=u"%s - 签到失败%s" % (
tpl['sitename'], u' 已停止' if disabled else u""),
text=u"""
您的 %(sitename)s [ %(siteurl)s ] 签到任务，执行 %(cnt)d次 失败。%(disable)s
下一次重试在一天之后，为防止签到中断，给您发送这份邮件。
访问： http://%(domain)s/task/%(taskid)s/log 查看日志。
""" % dict(
sitename=tpl['sitename'] or u'未命名',
siteurl=tpl['siteurl'] or u'',
cnt=task['last_failed_count'] + 1,
disable=u"因连续多次失败，已停止。" if disabled else u"",
domain=config.domain,
taskid=task['id'],
), async=True)
except Exception as e:
logging.error('send mail error: %r', e)
logger.error('taskid:%d tplid:%d failed! %r %.4fs', task['id'], task['tplid'], e, time.time()-start)
raise gen.Return(False)
raise gen.Return(True)
def task_failed(self, task, user, tpl, e):
pass
