def publish_task(project_id, obj_info):
"""发布任务数量, 异步任务回写"""
# 同步至数据库.
# 更新obj_info信息. object_url='', object_text=''
obj_value_info = [value for value in obj_info.values()]
from flask import request
from config.config import VERSION, BASE_API_URL
headers = get_headers()
base_url = BASE_API_URL + VERSION
url = os.path.join(base_url, 'upload/add_task')
circle_count = 500
circle_num = math.ceil(float(len(obj_value_info)) / circle_count)
# thread_pool_num = (circle_num <= 10) and circle_num or 20
# 保证有序性.
pivot_pos = 0
future_tasks = []
# print('长度 %s' % len(obj_value_info))
for _ in range(int(circle_num)):
# print(pivot_pos, pivot_pos+circle_count)
_obj_info = obj_value_info[pivot_pos:pivot_pos + circle_count]
# print('截取长度 %s' % len(_obj_info))
process = executor.submit(_add_task_thread, url, headers,
request.cookies, project_id, _obj_info)
future_tasks.append(process)
pivot_pos += circle_count
def tagging_sampling_check():
'''
抽检任务
:return:
'''
resp = requests.post(url=get_url(),
cookies=request.cookies,
json=request.json,
headers=get_headers())
filter_info = {
'filter_key': 'cache_id',
'filter_value': ['object_url', 'user_resp']
}
return req_util(resp,
Response,
filter_fields=['content'],
filter_info=filter_info)
def tagging_get():
'''
获取任务
:return:
'''
resp = requests.post(url=get_url(),
cookies=request.cookies,
json=request.json,
headers=get_headers())
filter_info = {
'filter_key': 'task_id',
'filter_value': ['object_url'],
'field': 'filter_obj.task_id == %s'
}
return req_util(resp,
Task,
filter_fields=['content'],
filter_info=filter_info)
def after_request(response):
headers = get_headers()
api_url = request.base_url.split(VERSION)[-1]
base_url = BASE_API_URL + VERSION
url = base_url + api_url
for rule in app.url_map._rules:
if isinstance(rule, Rule):
rule_ = rule.rule.split(VERSION)[-1]
if rule_ == api_url:
return response
if api_url == '/login':
resp = requests.post(url=url, cookies=request.cookies, data=request.form,
headers=headers)
resp_data = json.loads(resp.content)
response = jsonify(resp_data)
for k, v in resp.cookies.items():
response.set_cookie(k, v)
return response
else:
if request.method == 'POST':
resp = requests.post(url=url, cookies=request.cookies, json=request.json,
headers=headers)
elif request.method == 'GET':
resp = requests.get(url=url, cookies=request.cookies, headers=headers)
else:
return jsonify(code=404)
resp_data = json.loads(resp.content)
return jsonify(resp_data)
def tagging_submit():
'''
提交任务
:return:
'''
if not request.json:
return jsonify(code=10001, msg='未传递参数')
id = request.json.get('id', None)
user_resp = request.json.get('user_resp', None)
not_tagging = request.json.get('not_tagging', None)
edit_submit = request.json.get('edit_submit', None)
if id is None or (user_resp is None and not_tagging is None):
return jsonify(code=10001, msg='未传递参数')
data_dict = request.json
if data_dict.get('user_resp', None):
data_dict['user_resp'] = {
k: [1 for i in v]
for k, v in user_resp.items()
}
resp = requests.post(url=get_url(),
cookies=request.cookies,
json=request.json,
headers=get_headers())
if resp.status_code != 200:
return jsonify(resp.content)
resp_data = json.loads(resp.content)
if resp_data.get('code') != 200:
return jsonify(resp_data)
if edit_submit:
response = db.session.query(Response).filter(
Response.response_id == id).first()
if response is None:
return jsonify(code=10001, msg='此回答不存在')
response.user_resp = json.dumps(user_resp)
else:
resp_content = json.loads(resp.content)
if resp_content.get('code') != 200:
return jsonify(resp_data)
task = Task.query.filter_by(
task_id=resp_content.get('task_id')).first()
if task is None:
return jsonify(code=10001, msg='此任务不存在')
response = Response()
response.response_id = resp_content.get('response')
response.cache_id = id
response.object_url = task.object_url
response.task_id = resp_content.get('task_id')
response.user_resp = json.dumps(user_resp)
db.session.add(response)
db.session.commit()
return jsonify(resp_data)
def upload_page(req_json):
up_type = req_json.get("upload_type")
project_id = req_json.get("project_id")
requirement_id = req_json.get("requirement_id")
page = int(req_json.get("page", 1))
per_page = int(req_json.get("per_page", 10))
# if requirement_id:
# task = PlatformTask.get_instance(requirement_id)
# if not task:
# return jsonify(code=PARAMS_NOT_PROVIDED, msg="无效的参数")
# if task.creator_id != current_user.id:
# return jsonify(code=OPERATOR_ERROR, msg="非法操作")
#
# local_status = LocalStatus.query.filter_by(category='task', name='delete').first()
# if task.status == local_status.id:
# return jsonify(code=OPERATOR_ERROR, msg="需求已删除")
#
# if project_id:
# project = Project.get_instance(project_id)
# if not project:
# return jsonify(code=PARAMS_NOT_PROVIDED, msg="无效的参数")
# if current_user.platform != project.platform:
# return jsonify(code=OPERATOR_ERROR, msg="非法操作")
req_json["check_type"] = "upload_page"
from flask import request
from config.config import VERSION, BASE_API_URL
headers = get_headers()
base_url = BASE_API_URL + VERSION
url = os.path.join(base_url, 'upload/check_legal')
res = requests.post(url=url,
cookies=request.cookies,
json=req_json,
headers=headers)
print('-----------------------------------')
print(res.text, type(res.text))
print('-----------------------------------')
res_json = json.loads(res.text)
if res_json.get("code") != 200:
return res
upload_tmp_url_index = UPLOAD_TMP_URLS_INDEX.format(up_type=up_type,
id=(project_id
or requirement_id))
object_info = xredis.lrange(upload_tmp_url_index, (page - 1) * per_page,
page * per_page - 1)
total = xredis.llen(upload_tmp_url_index)
ret_info = []
pos = (page - 1) * per_page
for info in list(object_info):
info = eval(info)
object_type = info.get("object_type")
if object_type in ("tracking", ):
object_url = eval(info.get("object_url"))
main = object_url.get("main", {})
base_url = main.get("base_url", '')
pic_1 = main.get("picture", [''])[0]
object_url = os.path.join(base_url, pic_1)
info["object_url"] = os.path.join(get_endpoint, object_url)
else:
object_url = info.get("object_url")
if object_url and ("http" not in object_url):
abs_url = "{0}/{1}".format(get_endpoint, object_url)
info["object_url"] = abs_url
info["id"] = pos
pos += 1
ret_info.append(info)
return jsonify(code=200, data=ret_info, total=total)
def publish_batch(req_json):
batch_id = req_json.get("img_batch_id")
# batch_obj = ImgBatch.query.filter_by(id=batch_id).first()
# if not batch_obj:
# return jsonify(code=PARAMS_NOT_PROVIDED, msg="无效的批次id")
#
# published_id = LocalStatus.query.filter_by(category="batch",
# en_name="published").first().id
# if batch_obj.status == published_id:
# return jsonify(code=PARAMS_NOT_PROVIDED, msg="批次已发布")
#
# local_tmp_path = get_local_tmp_path("requirement", project_id=None,
# requirement_id=batch_obj.requirement_id)
#
# task = PlatformTask.get_instance(batch_obj.requirement_id)
# if task.creator_id != current_user.id:
# return jsonify(code=OPERATOR_ERROR, msg="操作非法")
#
# active_status = LocalStatus.query.filter_by(category="task",
# name="active").first()
# if task.status != active_status.id:
# return jsonify(code=PARAMS_NOT_PROVIDED, msg="需求未审核通过, 不允许发布")
#
# # 需求已发布项目, 更改批次状态为发布.
# if batch_obj.project_id:
# batch_obj.update(**{"status": published_id})
# return jsonify(code=200, msg="更新状态成功")
#
# # 创建项目
# protect_status = LocalStatus.query.filter_by(category="project",
# name="protect").first()
# if not current_user.platform:
# platform = User.query.filter(User.id==current_user.id).first().platform
# else:
# platform = current_user.platform
# request.json["name"] = task.name # 项目名称
# request.json["status"] = protect_status.id # 项目状态 默认保护
# request.json["finish_time"] = date2str(datetime.now() + timedelta(7))
# request.json["requirement_name"] = task.creator.username or task.creator.email
# request.json["batch_id"] = batch_obj.id
# request.json["temp_id"] = task.temp_id
# request.json["platform"] = platform
# request.json["demand_user_id"] = current_user.id
#
# del request.json["img_batch_id"]
# 创建项目
# pro_obj = sql_create(Project, ProjectCreateForm) # 同步执行.
#
# data = json.loads(pro_obj.data)
# project_id = data["data"]["id"]
#
# active_status = LocalStatus.query.filter_by(category="batch",
# name="published").first()
# batch_obj.update(**{"project_id": project_id, "status": active_status.id})
from config.config import VERSION, BASE_API_URL
headers = get_headers()
base_url = BASE_API_URL + VERSION
url = os.path.join(base_url, 'upload/check_legal')
req_json["check_type"] = "publish_batch"
res = requests.post(url, json=req_json, headers=headers)
res_json = json.loads(res.text)
if res_json.get("code") != 200:
return res
try:
project_id = res_json.get("data").get("project_id")
requirement_id = res_json.get("data").get("requirement_id")
upload_tmp_batch_url = UPLOAD_TMP_BATCH_URLS.format(
up_type="requirement", id=requirement_id, batch_id=batch_id)
obj_urls = xredis.hgetall(upload_tmp_batch_url)
publish_task(project_id, obj_urls)
xredis.delete(upload_tmp_batch_url)
except Exception as e:
Logger.error("发布任务异常 %s" % e)
return jsonify(code=1, msg="发布任务异常")
return jsonify(code=200, msg="批次发布成功")
def confirm_process(req_json):
"""确认上传"""
filename = req_json.get("file_name")
upload_type = req_json.get("upload_type")
project_id = req_json.get('project_id')
requirement_id = req_json.get('requirement_id')
# if upload_type == "requirement":
# obj = PlatformTask.get_instance(id_=requirement_id)
# if not obj:
# return jsonify(code=DATA_NOT_FOUND, msg="无效的需求id")
# if obj.creator_id != current_user.id:
# return jsonify(code=OPERATOR_ERROR, msg="非法操作")
#
# if upload_type == "project":
# obj = Project.get_instance(_id=project_id)
# if not obj:
# return jsonify(code=DATA_NOT_FOUND, msg="无效的项目id")
# if current_user.platform != obj.platform:
# return jsonify(code=OPERATOR_ERROR, msg="非法操作")
# 合法检查
upload_tmp_file = UPLOAD_TMP_FILE.format(up_type=upload_type,
id=(project_id or requirement_id))
upload_tmp_url = UPLOAD_TMP_URLS.format(up_type=upload_type,
id=(project_id or requirement_id))
upload_tmp_url_index = UPLOAD_TMP_URLS_INDEX.format(up_type=upload_type,
id=(project_id
or requirement_id))
req_json["check_type"] = "confirm_process"
req_json["counts"] = xredis.llen(upload_tmp_url_index)
# res = requests.post("check", json=req_json)
# res_json = json.loads(res.text())
# if res_json.get("code") != 200:
# return res
from flask import request
from config.config import VERSION, BASE_API_URL
headers = get_headers()
# api_url = request.base_url.split(VERSION)[-1]
base_url = BASE_API_URL + VERSION
url = os.path.join(base_url, 'upload/check_legal')
res = requests.post(url=url,
cookies=request.cookies,
json=req_json,
headers=headers)
print('-----------------------------------')
print(res.text, type(res.text))
print('-----------------------------------')
res_json = json.loads(res.text)
if res_json.get("code") != 200:
return res
local_path = get_local_tmp_path(upload_type, project_id, requirement_id)
# 类型检查
img_type, file_type, compress_type, _ = get_upload_file_type()
# # 确认提交文件存在.
# if not check_file_exist(local_path, filename):
# return jsonify(code=FILE_NOT_EXIST, msg='文件不存在')
# # 删除不在指定名称列表内文件.
# delete_not_exist_file(local_path, filename)
base_dir = os.path.dirname(local_path)
if upload_type == "requirement":
# 生成批次, 重命名tmp文件
# batch = ImgBatch.create(**{"desc": req_json.get("desc", ""),
# "status": BatchImgStatus.WAITING_PUBLISH.value,
# "img_list": "",
# "requirement_id": requirement_id}
# )
# 更新批次数量
# success_count = xredis.llen(upload_tmp_url_index)
# batch.update(**{"counts": success_count})
# 重命名tmp文件为批次id
batch_id = res_json.get("data").get("batch_id")
os.rename(local_path, os.path.join(base_dir, str(batch_id)))
xredis.delete(upload_tmp_file)
xredis.delete(upload_tmp_url_index)
upload_tmp_batch_urls = UPLOAD_TMP_BATCH_URLS.format(
up_type=upload_type,
id=(project_id or requirement_id),
batch_id=batch_id)
xredis.rename(upload_tmp_url, upload_tmp_batch_urls)
if upload_type == "project":
try:
obj_urls = xredis.hgetall(upload_tmp_url)
publish_task(project_id, obj_urls)
xredis.delete(upload_tmp_file)
xredis.delete(upload_tmp_url)
xredis.delete(upload_tmp_url_index)
except Exception as e:
Logger.error("发布任务异常 %s" % e)
return jsonify(code=1, msg="发布任务异常")
# 文件保存, 图片及压缩包删除.
for name in filename:
_type = name.split('.')[-1]
if _type in file_type:
os.rename(local_path,
os.path.join(base_dir, date2str(datetime.now())))
if os.path.exists(local_path):
shutil.rmtree(local_path)
return jsonify(code=200, msg="确认成功")
def uploaded_check(uploaded_files, req_form):
"""
上传文件
成功数, 失败数, 错误数.
"""
req_dict = req_form.to_dict()
up_type = req_dict.get("upload_type")
project_id = req_dict.get("project_id")
requirement_id = req_dict.get("requirement_id")
# 判断需求状态是否允许上传
# task = PlatformTask.get_instance(requirement_id)
# status_list = LocalStatus.query.filter_by(category="task").all()
# accept_status = [item.id for item in status_list if item.en_name in ("active", "nonactivated")]
#
# if requirement_id:
# if not task:
# return jsonify(code=PARAMS_NOT_PROVIDED, msg="无效的需求id")
# if task.status not in accept_status:
# return jsonify(code=DATA_NOT_FOUND, msg="需求状态, 不允许上传文件")
# if task.creator_id != current_user.id:
# return jsonify(code=OPERATOR_ERROR, msg="操作非法")
# local_status = LocalStatus.query.filter_by(category='task', name='delete').first()
# if task.status == local_status:
# return jsonify(code=OPERATOR_ERROR, msg="需求已删除")
#
# if project_id:
# project = Project.get_instance(project_id)
# if not project:
# return jsonify(code=PARAMS_NOT_PROVIDED, msg="无效的参数")
# if current_user.platform != project.platform:
# return jsonify(code=OPERATOR_ERROR, msg="非法操作")
req_dict["check_type"] = "uploaded_check"
# res = requests.post("check", json=req_dict)
# 文件缓存隔离级别为: 单个项目/需求, 每一次上传.
# 合法检查
from flask import request
from config.config import VERSION, BASE_API_URL
headers = get_headers()
# api_url = request.base_url.split(VERSION)[-1]
base_url = BASE_API_URL + VERSION
url = os.path.join(base_url, 'upload/check_legal')
res = requests.post(url=url,
cookies=request.cookies,
json=req_dict,
headers=headers)
print('-----------------------------------')
print(res.text, type(res.text))
print('-----------------------------------')
res_json = json.loads(res.text)
if res_json.get("code") != 200:
return res
upload_tmp_file = UPLOAD_TMP_FILE.format(up_type=up_type,
id=(project_id or requirement_id))
upload_tmp_url = UPLOAD_TMP_URLS.format(up_type=up_type,
id=(project_id or requirement_id))
upload_tmp_url_index = UPLOAD_TMP_URLS_INDEX.format(up_type=up_type,
id=(project_id
or requirement_id))
# 图片临时存放位置
local_tmp_path = get_local_tmp_path(up_type, project_id, requirement_id)
# 文件合法格式
img_type, file_type, compress_type, _ = get_upload_file_type()
# 文件成功数, 失败数, 重复数.
success_file_num, failure_file_num, repeat_file_num = 0, 0, 0
error_content = []
repeat_content = []
for f in uploaded_files:
file_name = f.filename
_name, _type = os.path.splitext(file_name)
ext = _type.strip('.').lower()
# 名称检查
is_legal, res_info = check_name(_name)
if not is_legal:
error_content.append({"file": file_name, "msg": '文件名称不合法'})
Logger.error("%s 文件名称不合法" % file_name)
failure_file_num += 1
continue
# 类型检查
if ext not in (img_type + file_type + compress_type):
error_content.append({"file": file_name, "msg": '文件后缀不合规'})
Logger.error("%s 文件后缀不合规" % file_name)
failure_file_num += 1
continue
file_info = xredis.hgetall(upload_tmp_file)
# 重名检查
if file_name in list(file_info.keys()):
error_content.append({"file": file_name, "msg": '文件名重复'})
Logger.error("%s 文件已存在, 文件名称重复" % file_name)
repeat_file_num += 1
continue
# 存放路径.
save_path = os.path.join(local_tmp_path, f.filename)
# 保存文件, 计算md5值, 判断文件是否已存在
if ext in img_type:
md5_code = save_ret_img_md5(f, save_path)
if ext in file_type:
md5_code = save_ret_file_md5(f, save_path)
if md5_code not in list(file_info.values()):
xredis.hset(upload_tmp_file, file_name, md5_code)
else:
os.remove(save_path)
error_content.append({"file": file_name, "msg": '文件已存在, 内容相同'})
Logger.error("%s 文件已存在, 内容相同" % file_name)
repeat_file_num += 1
continue
# 图片校验
if ext in img_type:
res = check_img_format(save_path)
if res is False:
os.remove(save_path)
xredis.hdel(upload_tmp_file, file_name)
Logger.error("%s 图片过大不合法" % file_name)
failure_file_num += 1
continue
id_ = requirement_id or project_id
cdn_path = get_cdn_path(req_dict={
"upload_type": up_type,
"id": id_
},
filename=file_name)
with open(save_path, 'rb') as pic_f:
upload_img(pic_f, cdn_path)
cdn_info = {'object_url': cdn_path, 'object_type': "img"}
xredis.hset(upload_tmp_url, cdn_path, str(cdn_info))
l_list = {
"object_type": "img",
"object_url": cdn_path,
"object_text": "",
"object_key": cdn_path
}
xredis.lpush(upload_tmp_url_index, str(l_list))
# 文件校验
if ext in file_type:
res, _, error_urls, repeat_urls = check_file_format(
save_path, upload_tmp_url, upload_tmp_url_index)
if res is True:
error_content.extend(error_urls)
repeat_content.extend(repeat_urls)
success_file_num += 1
else:
error_content.extend(error_urls)
repeat_content.extend(repeat_urls)
failure_file_num += 1
xredis.hdel(upload_tmp_file, file_name)
file_list = xredis.hgetall(upload_tmp_file)
object_info = xredis.lrange(upload_tmp_url_index, 0, 9)
ret_info = []
pos = 0
for info in list(object_info):
info = eval(info)
object_type = info.get("object_type")
if object_type in ("tracking", ):
object_url = eval(info.get("object_url"))
main = object_url.get("main", {})
base_url = main.get("base_url", '')
pic_1 = main.get("picture", [''])[0]
object_url = os.path.join(base_url, pic_1)
info["object_url"] = os.path.join(get_endpoint, object_url)
else:
object_url = info.get("object_url")
if object_url and ("http" not in object_url):
abs_url = "{0}/{1}".format(get_endpoint, object_url)
info["object_url"] = abs_url
info["id"] = pos
pos += 1
ret_info.append(info)
total = xredis.llen(upload_tmp_url_index)
data = {
"file_list": list(file_list.keys()),
"object_urls": ret_info,
"error_content": error_content,
"repeat_content": repeat_content,
"success_file_num": success_file_num,
"failure_file_num": failure_file_num,
"repeat_file_num": repeat_file_num
}
# 设定缓存过期时间, 1小时.
xredis.expire(upload_tmp_url, 60 * 60)
xredis.expire(upload_tmp_url_index, 60 * 60)
xredis.expire(upload_tmp_file, 60 * 60)
return jsonify(code=200,
msg="文件上传成功",
content={
"data": data,
"total": total
})
