def send_url_etcd(urllist, form): e = ETCD() online_worker = e.get_online_worker() if not online_worker: return ServerError(msg="没有可以处理链接的worker节点,请确认链接正确和相应worker在线") img_url_roots = [] for online in online_worker.values(): img_url_roots += online.keys() taskuuid = str(uuid.uuid4()) mlog = { "taskuuid": taskuuid, "handlemod": form.handlemod.data, "username": "******" + form.username.data, "remark": form.remark.data, "createtime": datetime.datetime.now(), "urls": [] } for imgurl in urllist: flag = 0 imgurl = imgurl.rstrip('\n').rstrip('\r').replace(' ', '') for url_root in img_url_roots: if re.match(url_root, imgurl): flag = 1 break urllog = {"url": imgurl, "taskinfo": []} if flag == 0: urllog["taskinfo"].append({"status": "warning"}) mlog["urls"].append(urllog) m = Mongo() mdb = m.client[MONGO_DATABASE] mdb[MONGO_COLLECTION].insert_one(mlog) # 将链接发送到etcd e.send_url_ectd(urllist, form.handlemod.data, taskuuid) time.sleep(1) # 等1秒后,获取返回结果后返回 taskinfo = mdb[MONGO_COLLECTION].find_one({"taskuuid": taskuuid}) if taskinfo: # 处理Mongodb时区问题 for urlinfo in taskinfo["urls"]: for tinfo in urlinfo['taskinfo']: if tinfo.get('updatetime'): tinfo['updatetime'] = tinfo[ 'updatetime'] + datetime.timedelta(hours=8) tinfo['updatetime'] = tinfo['updatetime'].strftime( '%Y-%m-%d %H:%M:%S') print(taskinfo) result = {'taskid': taskinfo["taskuuid"], 'urlsinfo': taskinfo["urls"]} return Success(msg=json.dumps(result)) else: return APIException(msg="获取任务处理历史出错,任务id:" + taskuuid)
def deadhistory(): page = int(request.args.get("page", 1)) pagelimit = 20 m = Mongo() mdb = m.client[MONGO_DATABASE] historytask = mdb[DEADLINK_COLLECTION].find({}, {"username": 1, "handlemod": 1, "createtime": 1, "webtaskinfo": 1, "remark": 1, "taskuuid": 1}).sort("createtime", pymongo.DESCENDING).\ skip(pagelimit*(page - 1)).limit(pagelimit) taskcounts = mdb[DEADLINK_COLLECTION].find().count() pageination = get_pagination(taskcounts, pagelimit, page) return render_template('deadlink/deadhistory.html', historytask=historytask, pageination=pageination)
def flush_cdn_taskuuid(): if request.args.get("taskuuid"): try: m = Mongo() mdb = m.client[MONGO_DATABASE] taskinfo = mdb[MONGO_COLLECTION].find_one( {"taskuuid": request.args.get("taskuuid")}) imageurls = '' for taski in taskinfo["urls"]: imageurls += taski["url"] + '\n' result = send_cdn(imageurls) return jsonify(result) except: return jsonify(traceback.format_exc()) else: return jsonify("请发送taskuuid")
def send_deadlink_etcd(urllist, form): e = ETCD() online_worker = e.get_online_worker() if not online_worker: flash("没有可以处理链接的worker节点,请确认链接正确和相应worker在线") return False img_url_roots = [] for online in online_worker.values(): img_url_roots += online.keys() taskuuid = str(uuid.uuid4()) mlog = { "taskuuid": taskuuid, "username": current_user.username, "remark": form.remark.data, "createtime": datetime.now(), "urls": [] } for deadlink in urllist: flag = 0 deadlink = deadlink.rstrip('\n').rstrip('\r').replace(' ', '') for url_root in img_url_roots: if re.match(url_root, deadlink): flag = 1 break urllog = {"url": deadlink, "taskinfo": []} if flag == 0: flash(deadlink + ":没有可以处理的worker,请确认链接格式") urllog["taskinfo"].append({"status": "warning"}) mlog["urls"].append(urllog) m = Mongo() mdb = m.client[MONGO_DATABASE] mdb[DEADLINK_COLLECTION].insert_one(mlog) # 将链接发送到etcd # e.send_url_ectd(urllist, form.handlemod.data, taskuuid) for deadlink in urllist: writekey = DEADLINKSDIR + str(uuid.uuid4()) lease = e.client.lease(30) task = { "taskuuid": taskuuid, "url": deadlink.replace('\n', '').replace('\r', '').replace(' ', '') } e.client.put(writekey, json.dumps(task), lease) return True
def deadhistorydetails(): page = int(request.args.get("page", 1)) pagelimit = 100 form = TaskInfoForm(request) if form.validate(): m = Mongo() mdb = m.client[MONGO_DATABASE] # taskinfo = mdb[DEADLINK_COLLECTION].find_one({"taskuuid": form.taskid.data}) tasks = mdb[DEADLINK_COLLECTION].aggregate( pipeline=[{ "$match": { 'taskuuid': form.taskid.data } }, { "$sort": { "urls": 1 } }, { "$project": { "urls": 1, "_id": 0 } }, { "$unwind": "$urls" }, { "$skip": pagelimit * (page - 1) }, { "$limit": pagelimit }]) taskinfo = [task['urls'] for task in tasks] print(taskinfo) if taskinfo: # 处理Mongodb时区问题 for urlinfo in taskinfo: for tinfo in urlinfo['taskinfo']: if tinfo.get('updatetime'): tinfo['updatetime'] = tinfo[ 'updatetime'] + datetime.timedelta(hours=8) # 获取分页信息 counts = mdb[DEADLINK_COLLECTION].aggregate(pipeline=[{ "$match": { 'taskuuid': form.taskid.data } }, { "$project": { 'count': { "$size": "$urls" } } }]) count = 1 for c in counts: count = c["count"] pageination = get_pagination(count, pagelimit, page) return render_template('deadlink/historydetails.html', taskinfo=taskinfo, taskid=form.taskid.data, pageination=pageination) else: flash("没有查询到任务相关详情信息") return redirect(url_for('web.deadhistory')) else: flash("任务id错误") return redirect(url_for('web.deadhistory'))