def upload_page():
"""
***上传文件, 文件分页***
***可以根据upload_type 进行逻辑判断.***
*** 路径 /api/v1.0.0/upload/uploads/page***
:
path: ""
**参数 request.json**
:
upload_type: project # 上传类型
project_id: 100 # 项目id
requirement_id: 8 # 需求id
page: 1
per_page: 20
**返回值 **
:
{
"code": 200,
"msg": "处理中"
}
"""
from config import Logger
try:
res = UploadController.upload_page(request.json)
except Exception as e:
Logger.error(e)
return res
def upload_confirm():
"""
***确认上传***
*** 路径 ***
:
path: "/api/v1.0.0/upload/upload_confirm"
**参数 request.json**
: {
upload_type: requirement
project_id: 100 # 项目id
requirement_id: 8 # 需求id
file_name: [name, name, name, ..., name]
}
**返回值 **
:
{
"code": 200,
"msg": "处理中"
}
"""
from config import Logger
try:
res = UploadController.confirm_upload(request.json)
except Exception as e:
Logger.error(e)
return res
class Uploader:
"""
Uploads files to Sharepoint.
"""
def __init__(self, params):
self._debug = params.debug
self._subsite = params.subsite
self._logger = Logger("spuppy", "logs", params.debug)
self._path = params.directory
self._files = params.files
self._out = params.out
def main(self):
"""
Orchestrate upload of a file using given configurations.
"""
spconf = SharepointConfig(self._subsite)
runtime = RuntimeConfig(self._debug)
fconf = FilesConfig(self._path, self._files, self._out)
verify = fconf.verify()
if self._debug:
self._logger.debug(spconf)
self._logger.debug(runtime)
self._logger.debug(fconf)
if verify:
for v in verify:
self._logger.error(v)
return
if self._debug:
self._logger.debug(fconf)
try:
sp = Sharepoint(fconf, spconf, runtime, self._logger)
sp.get_token()
if self._debug:
self._logger.debug(sp)
if not sp.verify_url():
return
if fconf.out_folder and not sp.add_folder():
return
sp.upload_files()
except Exception as e: # pylint: disable=broad-except
# This `except` is intentionally broad to capture it in
# the log for posterity. main exists anyway.
self._logger.exception(e)
def upload_func():
"""
***文件上传***
***可以根据upload_type 进行逻辑判断.***
*** 路径 /api/v1.0.0/upload/uploads***
:
path: ""
**参数 body.from-data**
:
upload_type: project # 上传类型
project_id: 100 # 项目id
requirement_id: 8 # 需求id
files: binary_file # 上传文件
file_type: png
**返回值 **
:
{
"code": 200,
"msg": "处理中"
}
"""
try:
req_dict = request.form.to_dict()
if req_dict.get("upload_type") in ("project", "requirement"):
res = UploadController.start_upload(request.files, request.form)
else:
headers = {'Referer': 'http://127.0.0.1:8008/15'}
api_url = request.base_url.split(VERSION)[-1]
base_url = BASE_API_URL + VERSION
url = base_url + api_url
files = {}
for req_file in request.files.getlist("files"):
save_path = os.path.join('upload', req_file.filename)
req_file.save(save_path)
files = {'files': open(save_path, 'rb')}
resp = requests.post(url=url,
cookies=request.cookies,
data=request.form,
files=files,
headers=headers)
resp_data = json.loads(resp.content)
return jsonify(resp_data)
except Exception as e:
Logger.error(e)
return res
def model_hash_verify():
"""
验证模型文件是否完整
:return:
"""
mode_md5 = Config.OCR.get('image_model_md5')
if not mode_md5:
# 跳过检测
return
md5 = file_md5(Config.IMAGE_MODEL_FILE)
if md5 != mode_md5:
os.remove(Config.IMAGE_MODEL_FILE)
Logger.error('模型文件校验失败，正在重新下载')
raise InvalidModelException()
return True
def _add_task_thread(url, headers, cookies, project_id, obj_info):
with app.app_context():
pos_obj_info = {}
for _index, _info in enumerate(obj_info):
info = eval(_info)
info["object_url"] = ''
info["object_text"] = ''
pos_obj_info[_index] = str(info)
data = {"project_id": project_id, "objects_info": pos_obj_info}
# print(data)
import time
# from datetime import datetime
# print("请求开始时间: %s" % datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
# begin_time =time.time()
res = requests.post(url=url,
headers=headers,
cookies=cookies,
json=data)
# pivot_time = time.time()
# print("请求结束时间: %s" % datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
# print("请求远端链接, 花费时间: %s" % (pivot_time-begin_time))
content = eval(res.text)
code = content.get('code')
if code == 200:
data = content.get('data')
for task_id, obj_url in zip(data, obj_info):
obj_url = eval(obj_url)
Task.create(
**{
"task_id": task_id,
"object_url": obj_url.get("object_url", "")
})
# print("写入数据库话费时间: %s" % (time.time()-pivot_time))
return True
else:
Logger.error("存在上传失败的数据段.")
return False
def delete_path():
"""
***删除路径***
***可以根据upload_type 进行逻辑判断.***
*** 路径 /api/v1.0.0/upload/uploads/delete_path***
:
path: ""
**参数 request.json**
:
upload_type: project # 上传类型
project_id: 100 # 项目id
requirement_id: 8 # 需求id
obj_urls: []
**返回值 **
:
{
"code": 200,
"msg": "处理成功"
}
"""
from config import Logger
try:
res = UploadController.delete_path(request.json)
except Exception as e:
Logger.error(e)
return res
# # todo 保留.
# @upload_bp.route('/upload/upload_delete', methods=['POST'])
# @verify_perm(code='upload_delete_doc')
# def upload_delete():
# """
# ***上传(文档/logo)删除***
#
# *** 路径 ***
# :
# path: "/api/v1.0.0/upload/upload_delete"
#
# **参数 request.json**
# : {
# path: "requirement_doc/2019/05/2275/状态码统计_1559202980.xls"
# requirement_id: 100
# platform_id:100
# }
# **返回值 **
# :
# {
# "code": 200,
# "msg": "处理中"
# }
# """
# from config import Logger
# try:
# res = UploadController.delete_cdn_path(request.json)
# except Exception as e:
# Logger.error(e)
# return res
def publish_batch(req_json):
batch_id = req_json.get("img_batch_id")
# batch_obj = ImgBatch.query.filter_by(id=batch_id).first()
# if not batch_obj:
# return jsonify(code=PARAMS_NOT_PROVIDED, msg="无效的批次id")
#
# published_id = LocalStatus.query.filter_by(category="batch",
# en_name="published").first().id
# if batch_obj.status == published_id:
# return jsonify(code=PARAMS_NOT_PROVIDED, msg="批次已发布")
#
# local_tmp_path = get_local_tmp_path("requirement", project_id=None,
# requirement_id=batch_obj.requirement_id)
#
# task = PlatformTask.get_instance(batch_obj.requirement_id)
# if task.creator_id != current_user.id:
# return jsonify(code=OPERATOR_ERROR, msg="操作非法")
#
# active_status = LocalStatus.query.filter_by(category="task",
# name="active").first()
# if task.status != active_status.id:
# return jsonify(code=PARAMS_NOT_PROVIDED, msg="需求未审核通过, 不允许发布")
#
# # 需求已发布项目, 更改批次状态为发布.
# if batch_obj.project_id:
# batch_obj.update(**{"status": published_id})
# return jsonify(code=200, msg="更新状态成功")
#
# # 创建项目
# protect_status = LocalStatus.query.filter_by(category="project",
# name="protect").first()
# if not current_user.platform:
# platform = User.query.filter(User.id==current_user.id).first().platform
# else:
# platform = current_user.platform
# request.json["name"] = task.name # 项目名称
# request.json["status"] = protect_status.id # 项目状态 默认保护
# request.json["finish_time"] = date2str(datetime.now() + timedelta(7))
# request.json["requirement_name"] = task.creator.username or task.creator.email
# request.json["batch_id"] = batch_obj.id
# request.json["temp_id"] = task.temp_id
# request.json["platform"] = platform
# request.json["demand_user_id"] = current_user.id
#
# del request.json["img_batch_id"]
# 创建项目
# pro_obj = sql_create(Project, ProjectCreateForm) # 同步执行.
#
# data = json.loads(pro_obj.data)
# project_id = data["data"]["id"]
#
# active_status = LocalStatus.query.filter_by(category="batch",
# name="published").first()
# batch_obj.update(**{"project_id": project_id, "status": active_status.id})
from config.config import VERSION, BASE_API_URL
headers = get_headers()
base_url = BASE_API_URL + VERSION
url = os.path.join(base_url, 'upload/check_legal')
req_json["check_type"] = "publish_batch"
res = requests.post(url, json=req_json, headers=headers)
res_json = json.loads(res.text)
if res_json.get("code") != 200:
return res
try:
project_id = res_json.get("data").get("project_id")
requirement_id = res_json.get("data").get("requirement_id")
upload_tmp_batch_url = UPLOAD_TMP_BATCH_URLS.format(
up_type="requirement", id=requirement_id, batch_id=batch_id)
obj_urls = xredis.hgetall(upload_tmp_batch_url)
publish_task(project_id, obj_urls)
xredis.delete(upload_tmp_batch_url)
except Exception as e:
Logger.error("发布任务异常 %s" % e)
return jsonify(code=1, msg="发布任务异常")
return jsonify(code=200, msg="批次发布成功")
def confirm_process(req_json):
"""确认上传"""
filename = req_json.get("file_name")
upload_type = req_json.get("upload_type")
project_id = req_json.get('project_id')
requirement_id = req_json.get('requirement_id')
# if upload_type == "requirement":
# obj = PlatformTask.get_instance(id_=requirement_id)
# if not obj:
# return jsonify(code=DATA_NOT_FOUND, msg="无效的需求id")
# if obj.creator_id != current_user.id:
# return jsonify(code=OPERATOR_ERROR, msg="非法操作")
#
# if upload_type == "project":
# obj = Project.get_instance(_id=project_id)
# if not obj:
# return jsonify(code=DATA_NOT_FOUND, msg="无效的项目id")
# if current_user.platform != obj.platform:
# return jsonify(code=OPERATOR_ERROR, msg="非法操作")
# 合法检查
upload_tmp_file = UPLOAD_TMP_FILE.format(up_type=upload_type,
id=(project_id or requirement_id))
upload_tmp_url = UPLOAD_TMP_URLS.format(up_type=upload_type,
id=(project_id or requirement_id))
upload_tmp_url_index = UPLOAD_TMP_URLS_INDEX.format(up_type=upload_type,
id=(project_id
or requirement_id))
req_json["check_type"] = "confirm_process"
req_json["counts"] = xredis.llen(upload_tmp_url_index)
# res = requests.post("check", json=req_json)
# res_json = json.loads(res.text())
# if res_json.get("code") != 200:
# return res
from flask import request
from config.config import VERSION, BASE_API_URL
headers = get_headers()
# api_url = request.base_url.split(VERSION)[-1]
base_url = BASE_API_URL + VERSION
url = os.path.join(base_url, 'upload/check_legal')
res = requests.post(url=url,
cookies=request.cookies,
json=req_json,
headers=headers)
print('-----------------------------------')
print(res.text, type(res.text))
print('-----------------------------------')
res_json = json.loads(res.text)
if res_json.get("code") != 200:
return res
local_path = get_local_tmp_path(upload_type, project_id, requirement_id)
# 类型检查
img_type, file_type, compress_type, _ = get_upload_file_type()
# # 确认提交文件存在.
# if not check_file_exist(local_path, filename):
# return jsonify(code=FILE_NOT_EXIST, msg='文件不存在')
# # 删除不在指定名称列表内文件.
# delete_not_exist_file(local_path, filename)
base_dir = os.path.dirname(local_path)
if upload_type == "requirement":
# 生成批次, 重命名tmp文件
# batch = ImgBatch.create(**{"desc": req_json.get("desc", ""),
# "status": BatchImgStatus.WAITING_PUBLISH.value,
# "img_list": "",
# "requirement_id": requirement_id}
# )
# 更新批次数量
# success_count = xredis.llen(upload_tmp_url_index)
# batch.update(**{"counts": success_count})
# 重命名tmp文件为批次id
batch_id = res_json.get("data").get("batch_id")
os.rename(local_path, os.path.join(base_dir, str(batch_id)))
xredis.delete(upload_tmp_file)
xredis.delete(upload_tmp_url_index)
upload_tmp_batch_urls = UPLOAD_TMP_BATCH_URLS.format(
up_type=upload_type,
id=(project_id or requirement_id),
batch_id=batch_id)
xredis.rename(upload_tmp_url, upload_tmp_batch_urls)
if upload_type == "project":
try:
obj_urls = xredis.hgetall(upload_tmp_url)
publish_task(project_id, obj_urls)
xredis.delete(upload_tmp_file)
xredis.delete(upload_tmp_url)
xredis.delete(upload_tmp_url_index)
except Exception as e:
Logger.error("发布任务异常 %s" % e)
return jsonify(code=1, msg="发布任务异常")
# 文件保存, 图片及压缩包删除.
for name in filename:
_type = name.split('.')[-1]
if _type in file_type:
os.rename(local_path,
os.path.join(base_dir, date2str(datetime.now())))
if os.path.exists(local_path):
shutil.rmtree(local_path)
return jsonify(code=200, msg="确认成功")
def uploaded_check(uploaded_files, req_form):
"""
上传文件
成功数, 失败数, 错误数.
"""
req_dict = req_form.to_dict()
up_type = req_dict.get("upload_type")
project_id = req_dict.get("project_id")
requirement_id = req_dict.get("requirement_id")
# 判断需求状态是否允许上传
# task = PlatformTask.get_instance(requirement_id)
# status_list = LocalStatus.query.filter_by(category="task").all()
# accept_status = [item.id for item in status_list if item.en_name in ("active", "nonactivated")]
#
# if requirement_id:
# if not task:
# return jsonify(code=PARAMS_NOT_PROVIDED, msg="无效的需求id")
# if task.status not in accept_status:
# return jsonify(code=DATA_NOT_FOUND, msg="需求状态, 不允许上传文件")
# if task.creator_id != current_user.id:
# return jsonify(code=OPERATOR_ERROR, msg="操作非法")
# local_status = LocalStatus.query.filter_by(category='task', name='delete').first()
# if task.status == local_status:
# return jsonify(code=OPERATOR_ERROR, msg="需求已删除")
#
# if project_id:
# project = Project.get_instance(project_id)
# if not project:
# return jsonify(code=PARAMS_NOT_PROVIDED, msg="无效的参数")
# if current_user.platform != project.platform:
# return jsonify(code=OPERATOR_ERROR, msg="非法操作")
req_dict["check_type"] = "uploaded_check"
# res = requests.post("check", json=req_dict)
# 文件缓存隔离级别为: 单个项目/需求, 每一次上传.
# 合法检查
from flask import request
from config.config import VERSION, BASE_API_URL
headers = get_headers()
# api_url = request.base_url.split(VERSION)[-1]
base_url = BASE_API_URL + VERSION
url = os.path.join(base_url, 'upload/check_legal')
res = requests.post(url=url,
cookies=request.cookies,
json=req_dict,
headers=headers)
print('-----------------------------------')
print(res.text, type(res.text))
print('-----------------------------------')
res_json = json.loads(res.text)
if res_json.get("code") != 200:
return res
upload_tmp_file = UPLOAD_TMP_FILE.format(up_type=up_type,
id=(project_id or requirement_id))
upload_tmp_url = UPLOAD_TMP_URLS.format(up_type=up_type,
id=(project_id or requirement_id))
upload_tmp_url_index = UPLOAD_TMP_URLS_INDEX.format(up_type=up_type,
id=(project_id
or requirement_id))
# 图片临时存放位置
local_tmp_path = get_local_tmp_path(up_type, project_id, requirement_id)
# 文件合法格式
img_type, file_type, compress_type, _ = get_upload_file_type()
# 文件成功数, 失败数, 重复数.
success_file_num, failure_file_num, repeat_file_num = 0, 0, 0
error_content = []
repeat_content = []
for f in uploaded_files:
file_name = f.filename
_name, _type = os.path.splitext(file_name)
ext = _type.strip('.').lower()
# 名称检查
is_legal, res_info = check_name(_name)
if not is_legal:
error_content.append({"file": file_name, "msg": '文件名称不合法'})
Logger.error("%s 文件名称不合法" % file_name)
failure_file_num += 1
continue
# 类型检查
if ext not in (img_type + file_type + compress_type):
error_content.append({"file": file_name, "msg": '文件后缀不合规'})
Logger.error("%s 文件后缀不合规" % file_name)
failure_file_num += 1
continue
file_info = xredis.hgetall(upload_tmp_file)
# 重名检查
if file_name in list(file_info.keys()):
error_content.append({"file": file_name, "msg": '文件名重复'})
Logger.error("%s 文件已存在, 文件名称重复" % file_name)
repeat_file_num += 1
continue
# 存放路径.
save_path = os.path.join(local_tmp_path, f.filename)
# 保存文件, 计算md5值, 判断文件是否已存在
if ext in img_type:
md5_code = save_ret_img_md5(f, save_path)
if ext in file_type:
md5_code = save_ret_file_md5(f, save_path)
if md5_code not in list(file_info.values()):
xredis.hset(upload_tmp_file, file_name, md5_code)
else:
os.remove(save_path)
error_content.append({"file": file_name, "msg": '文件已存在, 内容相同'})
Logger.error("%s 文件已存在, 内容相同" % file_name)
repeat_file_num += 1
continue
# 图片校验
if ext in img_type:
res = check_img_format(save_path)
if res is False:
os.remove(save_path)
xredis.hdel(upload_tmp_file, file_name)
Logger.error("%s 图片过大不合法" % file_name)
failure_file_num += 1
continue
id_ = requirement_id or project_id
cdn_path = get_cdn_path(req_dict={
"upload_type": up_type,
"id": id_
},
filename=file_name)
with open(save_path, 'rb') as pic_f:
upload_img(pic_f, cdn_path)
cdn_info = {'object_url': cdn_path, 'object_type': "img"}
xredis.hset(upload_tmp_url, cdn_path, str(cdn_info))
l_list = {
"object_type": "img",
"object_url": cdn_path,
"object_text": "",
"object_key": cdn_path
}
xredis.lpush(upload_tmp_url_index, str(l_list))
# 文件校验
if ext in file_type:
res, _, error_urls, repeat_urls = check_file_format(
save_path, upload_tmp_url, upload_tmp_url_index)
if res is True:
error_content.extend(error_urls)
repeat_content.extend(repeat_urls)
success_file_num += 1
else:
error_content.extend(error_urls)
repeat_content.extend(repeat_urls)
failure_file_num += 1
xredis.hdel(upload_tmp_file, file_name)
file_list = xredis.hgetall(upload_tmp_file)
object_info = xredis.lrange(upload_tmp_url_index, 0, 9)
ret_info = []
pos = 0
for info in list(object_info):
info = eval(info)
object_type = info.get("object_type")
if object_type in ("tracking", ):
object_url = eval(info.get("object_url"))
main = object_url.get("main", {})
base_url = main.get("base_url", '')
pic_1 = main.get("picture", [''])[0]
object_url = os.path.join(base_url, pic_1)
info["object_url"] = os.path.join(get_endpoint, object_url)
else:
object_url = info.get("object_url")
if object_url and ("http" not in object_url):
abs_url = "{0}/{1}".format(get_endpoint, object_url)
info["object_url"] = abs_url
info["id"] = pos
pos += 1
ret_info.append(info)
total = xredis.llen(upload_tmp_url_index)
data = {
"file_list": list(file_list.keys()),
"object_urls": ret_info,
"error_content": error_content,
"repeat_content": repeat_content,
"success_file_num": success_file_num,
"failure_file_num": failure_file_num,
"repeat_file_num": repeat_file_num
}
# 设定缓存过期时间, 1小时.
xredis.expire(upload_tmp_url, 60 * 60)
xredis.expire(upload_tmp_url_index, 60 * 60)
xredis.expire(upload_tmp_file, 60 * 60)
return jsonify(code=200,
msg="文件上传成功",
content={
"data": data,
"total": total
})
