def cron_check():
with scheduler.app.app_context():
try:
def dbs():
url = current_app.config.get('CRON_DB_URL')
db = records.Database(url)
db = db.get_connection() # 新加
return db
job_db = dbs()
job_arr = []
jobs = job_db.query("select id from apscheduler_jobs").all()
if jobs:
for item in jobs:
job_arr.append(item.id)
cifs = CronInfos.query.all()
if cifs:
for item in cifs:
if "cron_%s" % item.id not in job_arr:
item.status = -1
db.session.add(item)
db.session.commit()
except Exception as e:
trace_info = traceback.format_exc()
current_app.logger.error("==============")
current_app.logger.error(str(e))
current_app.logger.error(trace_info)
current_app.logger.error("==============")
wechat_info_err('定时任务发生严重错误', '返回信息:%s' % str(e))
return "ok"
def cron_do(cron_id):
with scheduler.app.app_context():
cif = CronInfos.query.get(cron_id)
if not cif:
jl = JobLog(cron_info_id=cron_id,content="定时任务不存在",create_time=get_now_time(),take_time=0)
db.session.add(jl)
db.session.commit()
else:
req_url = cif.req_url
if not req_url:
jl = JobLog(cron_info_id=cron_id, content="请求链接不存在", create_time=get_now_time(), take_time=0)
db.session.add(jl)
db.session.commit()
else:
if req_url.find('http') == -1:
jl = JobLog(cron_info_id=cron_id, content="请求链接有误，请检查一下", create_time=get_now_time(), take_time=0)
db.session.add(jl)
db.session.commit()
else:
t = time.time()
req = requests.get(req_url,timeout=2*60,headers={'user-agent':'xmb_cron'})
ret = req.text
try:
ret = req.json()
errcode = ret.get('errcode')
if errcode:
if int(errcode) !=0:
wechat_info_err('定时任务【%s】发生错误' % cif.task_name,'返回信息:%s' % json.dumps(ret,ensure_ascii=False))
except:
pass
if type(ret) == dict:
ret = json.dumps(ret,ensure_ascii=False)
jl = JobLog(cron_info_id=cron_id, content=ret, create_time=get_now_time(),take_time=time.time() - t)
db.session.add(jl)
db.session.commit()
return ""
def gen_status(*args, **kwargs):
try:
result = func(*args, **kwargs)
if type(result) == str:
return api_return(errcode=0, errmsg=result)
if type(result) == list or type(result) == dict:
return api_return(errcode=0, errmsg='success', data=result)
if type(result) == tuple:
if len(result) == 2:
errmsg = result[0]
if errmsg is None or errmsg == "":
errmsg = 'success'
return api_return(errcode=0, errmsg=errmsg, data=result[1])
else:
return api_return(errcode=result[0],
errmsg=result[1],
data=result[2])
except Exception as e:
error = str(e)
trace_info = traceback.format_exc()
wechat_info_err("小牛异步队列-BUG", trace_info)
return api_return(errcode=1, errmsg=error)
def cron_do(cron_id):
try:
with scheduler.app.app_context():
nows = get_now_time()
cif = CronInfos.query.get(cron_id)
if not cif:
jl = JobLog(cron_info_id=cron_id,content="定时任务不存在",create_time=nows,take_time=0)
db.session.add(jl)
db.session.commit()
else:
req_url = cif.req_url
if not req_url:
jl = JobLog(cron_info_id=cron_id, content="请求链接不存在", create_time=nows, take_time=0)
db.session.add(jl)
db.session.commit()
else:
if req_url.find('http') == -1:
jl = JobLog(cron_info_id=cron_id, content="请求链接有误，请检查一下", create_time=nows, take_time=0)
db.session.add(jl)
db.session.commit()
else:
try:
t = time.time()
req = requests.get(req_url,timeout=2*60,headers={'user-agent':'xmb_cron'})
ret = req.text
try:
ret = req.json()
except:
pass
if type(ret) == dict:
ret = json.dumps(ret,ensure_ascii=False)
error_keyword = configs('error_keyword')
if error_keyword:
error_keyword = error_keyword.replace('，', ',').split(',')
for item in error_keyword:
if item.strip().lower() in ret.lower():
wechat_info_err('定时任务【%s】发生错误' % cif.task_name, '返回信息:%s' % ret)
break
jl = JobLog(cron_info_id=cron_id, content=ret, create_time=nows,take_time=time.time() - t)
db.session.add(jl)
db.session.commit()
except Exception as e:
jl = JobLog(cron_info_id=cron_id, content="发生严重错误:%s" % str(e), create_time=nows, take_time=time.time() - t)
db.session.add(jl)
db.session.commit()
wechat_info_err('定时任务【%s】发生严重错误' % cif.task_name, '返回信息:%s' % str(e))
except Exception as e:
wechat_info_err('定时任务发生严重错误', '返回信息:%s' % str(e))
return ""
def celery_run(url,req_id,queue,priority,delay_time=0,post_json=None,is_fail_try=0,is_first=True,fail_counts=0,remark=''):
time1 = time.time()
try:
is_err = False
config = current_app.config.get('CONFIGS')
timeouts = 60*2
tt_list = config.get('retry_times')
if tt_list:
tt_list = tt_list.replace('，',',').split(',')
else:
tt_list = [30,60,180,900,1800,3600,43200]
headers = {
'User-Agent':'xiaoniu_tasks'
}
reqs = requests.Session()
if post_json:
rep = reqs.post(url,data=post_json,timeout=timeouts,headers=headers)
else:
rep = reqs.get(url, timeout=timeouts,headers=headers)
ret = rep.text
try:
ret = rep.json()
except:
pass
if type(ret) == dict:
ret = json.dumps(ret, ensure_ascii=False)
error_keyword = config.get('error_keyword')
if error_keyword and is_fail_try == 1:
error_keyword = error_keyword.replace('，', ',').split(',')
for item in error_keyword:
if item.strip().lower() in ret.lower():
is_err = True
wechat_info_err('定时任务【%s】发生错误' % req_id, '返回信息:%s' % ret)
break
if is_err is True:
#看是否得重试
if is_first is True:
is_first = False
if is_fail_try == 1:delay_time = int(tt_list[0])
else:
if is_fail_try== 1:
is_first = False
if int(delay_time) == int(tt_list[-1]):
is_fail_try = 0
delay_time = -1
else:
delay_time = int(tt_list[tt_list.index(str(delay_time)) + 1])
if delay_time !=-1:
celery_run.apply_async(queue=queue,priority=priority,countdown = int(delay_time),args=[url,req_id,queue,priority,delay_time,post_json,is_fail_try,is_first,fail_counts,remark])
time2 = time.time() - time1
rl = ReqLog.query.filter(ReqLog.req_id == req_id).first()
if not rl:
rl = ReqLog(req_url=url,req_id=req_id,post_json=post_json,respond=ret,create_time=get_now_time(),is_err=int(is_err),take_time=time2)
else:
rl.respond = "%s<br/><br/>%s" % (rl.respond,ret)
rl.update_time = get_now_time()
rl.is_err = int(is_err)
rl.take_time = time2
db.session.add(rl)
db.session.commit()
except Exception as e:
if fail_counts <=5:
fail_counts = fail_counts + 1
celery_run.apply_async(queue=queue,priority=priority,countdown=30*fail_counts,args=[url, req_id, queue,priority,delay_time, post_json, is_fail_try, is_first,fail_counts,remark])
# trace_info = traceback.format_exc()
# print(trace_info)
