def send_url_etcd(urllist, form):
e = ETCD()
online_worker = e.get_online_worker()
if not online_worker:
return ServerError(msg="没有可以处理链接的worker节点，请确认链接正确和相应worker在线")
img_url_roots = []
for online in online_worker.values():
img_url_roots += online.keys()
taskuuid = str(uuid.uuid4())
mlog = {
"taskuuid": taskuuid,
"handlemod": form.handlemod.data,
"username": "******" + form.username.data,
"remark": form.remark.data,
"createtime": datetime.datetime.now(),
"urls": []
}
for imgurl in urllist:
flag = 0
imgurl = imgurl.rstrip('\n').rstrip('\r').replace(' ', '')
for url_root in img_url_roots:
if re.match(url_root, imgurl):
flag = 1
break
urllog = {"url": imgurl, "taskinfo": []}
if flag == 0:
urllog["taskinfo"].append({"status": "warning"})
mlog["urls"].append(urllog)
m = Mongo()
mdb = m.client[MONGO_DATABASE]
mdb[MONGO_COLLECTION].insert_one(mlog)
# 将链接发送到etcd
e.send_url_ectd(urllist, form.handlemod.data, taskuuid)
time.sleep(1) # 等1秒后，获取返回结果后返回
taskinfo = mdb[MONGO_COLLECTION].find_one({"taskuuid": taskuuid})
if taskinfo:
# 处理Mongodb时区问题
for urlinfo in taskinfo["urls"]:
for tinfo in urlinfo['taskinfo']:
if tinfo.get('updatetime'):
tinfo['updatetime'] = tinfo[
'updatetime'] + datetime.timedelta(hours=8)
tinfo['updatetime'] = tinfo['updatetime'].strftime(
'%Y-%m-%d %H:%M:%S')
print(taskinfo)
result = {'taskid': taskinfo["taskuuid"], 'urlsinfo': taskinfo["urls"]}
return Success(msg=json.dumps(result))
else:
return APIException(msg="获取任务处理历史出错，任务id：" + taskuuid)
def deadhistory():
page = int(request.args.get("page", 1))
pagelimit = 20
m = Mongo()
mdb = m.client[MONGO_DATABASE]
historytask = mdb[DEADLINK_COLLECTION].find({}, {"username": 1, "handlemod": 1, "createtime": 1, "webtaskinfo": 1,
"remark": 1, "taskuuid": 1}).sort("createtime",
pymongo.DESCENDING).\
skip(pagelimit*(page - 1)).limit(pagelimit)
taskcounts = mdb[DEADLINK_COLLECTION].find().count()
pageination = get_pagination(taskcounts, pagelimit, page)
return render_template('deadlink/deadhistory.html',
historytask=historytask,
pageination=pageination)
def flush_cdn_taskuuid():
if request.args.get("taskuuid"):
try:
m = Mongo()
mdb = m.client[MONGO_DATABASE]
taskinfo = mdb[MONGO_COLLECTION].find_one(
{"taskuuid": request.args.get("taskuuid")})
imageurls = ''
for taski in taskinfo["urls"]:
imageurls += taski["url"] + '\n'
result = send_cdn(imageurls)
return jsonify(result)
except:
return jsonify(traceback.format_exc())
else:
return jsonify("请发送taskuuid")
def send_deadlink_etcd(urllist, form):
e = ETCD()
online_worker = e.get_online_worker()
if not online_worker:
flash("没有可以处理链接的worker节点，请确认链接正确和相应worker在线")
return False
img_url_roots = []
for online in online_worker.values():
img_url_roots += online.keys()
taskuuid = str(uuid.uuid4())
mlog = {
"taskuuid": taskuuid,
"username": current_user.username,
"remark": form.remark.data,
"createtime": datetime.now(),
"urls": []
}
for deadlink in urllist:
flag = 0
deadlink = deadlink.rstrip('\n').rstrip('\r').replace(' ', '')
for url_root in img_url_roots:
if re.match(url_root, deadlink):
flag = 1
break
urllog = {"url": deadlink, "taskinfo": []}
if flag == 0:
flash(deadlink + "：没有可以处理的worker，请确认链接格式")
urllog["taskinfo"].append({"status": "warning"})
mlog["urls"].append(urllog)
m = Mongo()
mdb = m.client[MONGO_DATABASE]
mdb[DEADLINK_COLLECTION].insert_one(mlog)
# 将链接发送到etcd
# e.send_url_ectd(urllist, form.handlemod.data, taskuuid)
for deadlink in urllist:
writekey = DEADLINKSDIR + str(uuid.uuid4())
lease = e.client.lease(30)
task = {
"taskuuid": taskuuid,
"url": deadlink.replace('\n', '').replace('\r',
'').replace(' ', '')
}
e.client.put(writekey, json.dumps(task), lease)
return True
def deadhistorydetails():
page = int(request.args.get("page", 1))
pagelimit = 100
form = TaskInfoForm(request)
if form.validate():
m = Mongo()
mdb = m.client[MONGO_DATABASE]
# taskinfo = mdb[DEADLINK_COLLECTION].find_one({"taskuuid": form.taskid.data})
tasks = mdb[DEADLINK_COLLECTION].aggregate(
pipeline=[{
"$match": {
'taskuuid': form.taskid.data
}
}, {
"$sort": {
"urls": 1
}
}, {
"$project": {
"urls": 1,
"_id": 0
}
}, {
"$unwind": "$urls"
}, {
"$skip": pagelimit * (page - 1)
}, {
"$limit": pagelimit
}])
taskinfo = [task['urls'] for task in tasks]
print(taskinfo)
if taskinfo:
# 处理Mongodb时区问题
for urlinfo in taskinfo:
for tinfo in urlinfo['taskinfo']:
if tinfo.get('updatetime'):
tinfo['updatetime'] = tinfo[
'updatetime'] + datetime.timedelta(hours=8)
# 获取分页信息
counts = mdb[DEADLINK_COLLECTION].aggregate(pipeline=[{
"$match": {
'taskuuid': form.taskid.data
}
}, {
"$project": {
'count': {
"$size": "$urls"
}
}
}])
count = 1
for c in counts:
count = c["count"]
pageination = get_pagination(count, pagelimit, page)
return render_template('deadlink/historydetails.html',
taskinfo=taskinfo,
taskid=form.taskid.data,
pageination=pageination)
else:
flash("没有查询到任务相关详情信息")
return redirect(url_for('web.deadhistory'))
else:
flash("任务id错误")
return redirect(url_for('web.deadhistory'))