def get(self: Resource, args: Dict[str,
Any]) -> Tuple[Dict[str, Any], int]:
"""
获取自定义容器list，分页，可以根据[custom_types]和[custom_states]过滤
"""
# get filtered list
filtered_list = {}
if args.get("custom_types"):
nlp_task_id_list = []
for nlp_task in args.get("custom_types").split(','):
if nlp_task == 'ner':
nlp_task = 'extract'
nlp_task_id_list.append(int(NlpTaskEnum[nlp_task]))
filtered_list.update(nlp_task_id_list=nlp_task_id_list)
if args.get("custom_states"):
filtered_list.update(custom_algorithm_status_list=[
status_str2int_mapper().get(status, int(StatusEnum[status]))
for status in args['custom_states'].split(',')
])
count, custom_algorithm_list = ModelCustomService(
).get_custom_algorithm_list_by_filter_in(offset=args["offset"],
limit=args["limit"],
args=filtered_list)
result = CustomAlgorithmSchema(many=True).dump(custom_algorithm_list)
return {
"message": "请求成功",
"result": result,
"count": count,
}, 200
def patch(self: Resource, args: typing.Dict, model_id: int,
model_evaluate_id: int) -> typing.Tuple[typing.Dict, int]:
"""
更新一条评估记录
"""
update_params = {}
if args.get("model_evaluate_state"):
update_params.update(evaluate_task_status=status_str2int_mapper()[
args["model_evaluate_state"]])
if args.get("model_evaluate_result"):
update_params.update(
evaluate_task_result=args["model_evaluate_result"])
if args.get("model_evaluate_name"):
update_params.update(
evaluate_task_name=args["model_evaluate_name"])
if args.get("model_evaluate_desc"):
update_params.update(
evaluate_task_desc=args["model_evaluate_desc"])
evaluate_task = ModelEvaluateService().update_evaluate_task_by_id(
evaluate_task_id=model_evaluate_id, args=update_params)
result = EvaluateTaskSchema().dump(evaluate_task)
return {
"message": "更新成功",
"result": result,
}, 200
def post(self: Resource, args) -> Tuple[Dict[str, Any], int]:
"""
更新自定义模型状态
"""
update_params = {"custom_algorithm_status": status_str2int_mapper()[args["custom_state"]]}
ModelCustomService().update_custom_algorithm_by_id(custom_algorithm_id=args["custom_id"], args=update_params)
return {
"message": "更新成功",
}, 201
def post(self: Resource,
args: typing.Dict) -> typing.Tuple[typing.Dict, int]:
"""
更新文档的处理状态
"""
update_params = {"doc_status": status_str2int_mapper()[args["convert_state"]]}
DocService().update_doc_by_id(doc_id=args["doc_id"],
args=update_params)
return {
"message": "更新成功",
}, 201
def update_mark_task_status(current_user: CurrentUser, task_id, args):
if args.get('task_result'):
args['mark_task_result'] = args.get('task_result')
del args['task_result']
if args.get('task_state'):
args['mark_task_status'] = status_str2int_mapper()[args['task_state']]
del args['task_state']
item = MarkTaskModel().update(task_id, **args)
schema = MarkTaskSchema
session.commit()
result = schema().dump(item)
return result
def update_user_task_status(current_user: CurrentUser, task_id, args):
if args.get('task_result'):
args['user_task_result'] = args.get('task_result')
del args['task_result']
if args.get('task_state'):
args['user_task_status'] = status_str2int_mapper()[args['task_state']]
del args['task_state']
item = UserTaskModel().update_by_annotator_id(current_user, task_id, **args)
MarkTaskModel().check_user_task_and_update_mark_task(task_id)
schema = UserTaskSchema
session.commit()
result = schema().dump(item)
return result
def put(self, args):
"""
修改模型的状态
"""
update_params = {}
if args.get("model_train_state"):
# 后端返回结果转换，失败时后端目前还返回failed
update_params.update(train_status=status_str2int_mapper()[args["model_train_state"]])
train_task = ModelTrainService().update_train_task_by_model_version(model_version=args["model_version"], is_check_train_terms=args["check_train_terms"], args=update_params)
result = TrainTaskSchema().dump(train_task)
return {
"message": "更新成功",
"result": result,
}, 201
def get_user_task_with_doc_and_doc_type(nlp_task_id,
current_user: CurrentUser, args):
q = session.query(UserTask, DocType, Doc) \
.join(MarkTask, MarkTask.mark_task_id == UserTask.mark_task_id) \
.join(MarkJob, MarkJob.mark_job_id == MarkTask.mark_job_id) \
.join(DocType, DocType.doc_type_id == MarkJob.doc_type_id) \
.join(Doc, Doc.doc_id == MarkTask.doc_id) \
.filter(
DocType.nlp_task_id == nlp_task_id,
~UserTask.is_deleted,
~MarkTask.is_deleted,
~Doc.is_deleted
)
# TODO
# 权限
if current_user.user_role in [
RoleEnum.manager.value, RoleEnum.guest.value
]:
q = q.filter(DocType.group_id.in_(current_user.user_groups))
elif current_user.user_role in [RoleEnum.reviewer.value]:
q = q.filter(
func.json_contains(MarkJob.reviewer_ids,
str(current_user.user_id)))
elif current_user.user_role in [RoleEnum.annotator.value]:
# q = q.filter(func.json_contains(MarkJob.annotator_ids, str(current_user.user_id)))
q = q.filter(UserTask.annotator_id == current_user.user_id)
if args.get('job_id'):
q = q.filter(MarkTask.mark_job_id == args['job_id'])
if args.get('doc_type_id'):
q = q.filter(MarkJob.doc_type_id == args['doc_type_id'])
if args['task_state']:
q = q.filter(MarkTask.mark_task_status ==
status_str2int_mapper().get(args['task_state']))
if args['query']:
q = q.filter(Doc.doc_raw_name.like(f'%{args["query"]}%'))
q = q.group_by(UserTask)
count = q.count()
processing_count = q.filter(
MarkTask.mark_task_status == int(StatusEnum.processing)).count()
if args['order_by'] and isinstance(args['order_by'], str):
if args['order_by'][1:] == 'task_id':
args['order_by'] = args['order_by'][0] + 'mark_task_id'
q = Common().order_by_model_fields(q, UserTask, [args['order_by']])
items = []
for user_task, doc_type, doc in q.offset(args['offset']).limit(
args['limit']).all():
user_task.doc = doc
user_task.doc_type = doc_type
items.append(user_task)
return count, count - processing_count, items
def put(self, args):
"""
修改模型训练的一个字段状态
"""
update_params = {}
if args.get("train_term_state"):
update_params.update(train_term_status=status_str2int_mapper()[args["train_term_state"]])
if args.get("train_term_result"):
update_params.update(train_term_result=args["train_term_result"])
train_term_task = ModelTrainService().update_train_term_by_model_version_and_doc_term_id(model_version=args["model_version"],
doc_term_id=args["doc_term_id"],
args=update_params)
result = TrainTermTaskSchema().dump(train_term_task)
return {
"message": "更新成功",
"result": result,
}, 201
def post(self: Resource, args: typing.Dict) -> typing.Tuple[typing.Dict, int]:
"""
message queue回调统一入口
"""
message = args['message']
logger.info(f"receive callback info from mq. response is: {json.dumps(args)}")
if message['business'] in [
'label', # 实体预标注
'classify_label', # 分类预标注
'relation_label', # 实体关系预标注
'wordseg_label' # 分词预标注
]:
update_params = {}
if args.get("task_state"):
if args['task_state'] == 'success': # 如果mq预标注返回成功，则初试状态是unlabel
update_params.update(mark_task_status=int(StatusEnum.unlabel))
else: # 如果mq预标注返回失败，则初试状态是fail
update_params.update(mark_task_status=int(StatusEnum.fail))
if args.get("task_result"):
update_params.update(mark_task_result=args["task_result"])
mark_task, user_task_list = MarkJobService()\
.update_mark_task_and_user_task_by_mark_task_id(mark_task_id=message["task_id"], args=update_params)
MarkJobService().update_mark_job_status_by_mark_task(mark_task=mark_task)
result = UserTaskSchema(many=True).dump(user_task_list)
return {
"message": "更新成功",
"result": result,
}, 201
elif message['business'] in [
'extract', # 实体抽取
'classify_extract', # 分类抽取
'relation_extract', # 实体关系抽取
'wordseg_extract' # 分词抽取
]:
update_params = {}
if args.get("task_state"):
update_params.update(predict_task_status=status_str2int_mapper()[args["task_state"]])
if args.get("task_result"):
update_params.update(predict_task_result=args["task_result"])
predict_task = PredictService().update_predict_task_by_id(predict_task_id=message["task_id"], args=update_params)
result = PredictTaskSchema().dump(predict_task)
return {
"message": "更新成功",
"result": result,
}, 201
def patch(self: Resource, args: typing.Dict,
job_id: int) -> typing.Tuple[typing.Dict, int]:
update_params = {}
if args.get("extract_job_name"):
update_params.update(predict_job_name=args["extract_job_name"])
if args.get("extract_job_desc"):
update_params.update(predict_job_desc=args["extract_job_desc"])
if args.get("extract_job_state"):
update_params.update(predict_job_status=status_str2int_mapper()[
args["extract_job_state"]])
predict_job = PredictService().update_predict_job_by_id(
predict_job_id=job_id, args=update_params)
result = PredictJobSchema().dump(predict_job)
return {
"message": "更新成功",
"result": result,
}, 200
def patch(self: Resource, args: typing.Dict,
task_id: int) -> typing.Tuple[typing.Dict, int]:
update_params = {}
if args.get("task_state"):
update_params.update(predict_task_status=status_str2int_mapper()[
args["task_state"]])
if args.get("task_result"):
update_params.update(predict_task_result=args["task_result"])
predict_task = TaskMachineService().update_predict_task_by_id(
predict_task_id=task_id, args=update_params)
result = PredictTaskSchema().dump(predict_task)
# 根据task更新整个predict_job的状态
TaskMachineService().update_predict_job_status_by_predict_task(
predict_task=predict_task)
return {
"message": "更新成功",
"result": result,
}, 200
def put(self, args):
"""
更新一条评估记录
"""
update_params = {}
if args.get("model_evaluate_state"):
update_params.update(evaluate_task_status=status_str2int_mapper()[
args["model_evaluate_state"]])
if args.get("model_evaluate_result"):
update_params.update(
evaluate_task_result=args["model_evaluate_result"])
evaluate_task = ModelEvaluateService().update_evaluate_task_by_id(
evaluate_task_id=args["model_evaluate_id"], args=update_params)
result = EvaluateTaskSchema().dump(evaluate_task)
return {
"message": "更新成功",
"result": result,
}, 201
def patch(
self: Resource,
args: typing.Dict,
model_id: int,
model_train_id: int
) -> typing.Tuple[typing.Dict, int]:
"""
修改模型的状态和结果
"""
update_params = {}
if args.get("model_train_state"): # 这里不考虑model_train_result因为新的表结构里没有这个列了
update_params.update(train_status=status_str2int_mapper()[args["model_train_state"]])
train_task = ModelTrainService().update_train_task_by_id(train_job_id=model_id, train_task_id=model_train_id,
is_check_train_terms=args["check_train_terms"], model_type = args["model_type"],
args=update_params)
result = TrainTaskSchema().dump(train_task)
return {
"message": "更新成功",
"result": result,
}, 200
def patch(
self: Resource,
args: typing.Dict,
model_id: int,
model_train_id: int,
train_term_id: int,
) -> typing.Tuple[typing.Dict, int]:
"""
修改模型训练的一个字段状态
"""
update_params = {}
if args.get("train_term_state"):
update_params.update(train_term_status=status_str2int_mapper()[args["train_term_state"]])
if args.get("train_term_result"):
update_params.update(train_term_result=args["train_term_result"])
train_term_task = ModelTrainService().update_train_task_term_by_id(train_term_task_id=train_term_id, args=update_params)
result = TrainTermTaskSchema().dump(train_term_task)
return {
"message": "更新成功",
"result": result,
}, 200
def get(self: Resource, args: typing.Dict):
order_by = args["order_by"][1:]
order_by_desc = True if args["order_by"][0] == "-" else False
filtered_list = {}
if args.get("task_state"):
filtered_list.update(predict_task_status=status_str2int_mapper()[
args["task_state"]])
count, predict_task_list = TaskMachineService(
).get_predict_task_list_by_predict_job_id(
predict_job_id=args["extract_job_id"],
search=args['query'],
order_by=order_by,
order_by_desc=order_by_desc,
offset=args['offset'],
limit=args['limit'],
current_user=self.get_current_user(),
args=filtered_list)
result = PredictTaskSchema(
many=True, exclude=('task_result', )).dump(predict_task_list)
return {
"message": "请求成功",
"result": result,
"count": count,
}, 200
def get_mark_task_with_doc_and_doc_type(self, nlp_task_id,
current_user: CurrentUser, args):
q = session.query(MarkTask, DocType, Doc) \
.join(MarkJob, MarkJob.mark_job_id == MarkTask.mark_job_id) \
.join(DocType, DocType.doc_type_id == MarkJob.doc_type_id) \
.join(Doc, Doc.doc_id == MarkTask.doc_id) \
.filter(
DocType.nlp_task_id == nlp_task_id,
~DocType.is_deleted,
~MarkTask.is_deleted,
~Doc.is_deleted
)
# TODO
# 权限
if current_user.user_role in [
RoleEnum.manager.value, RoleEnum.guest.value
]:
q = q.filter(DocType.group_id.in_(current_user.user_groups))
elif current_user.user_role in [RoleEnum.reviewer.value]:
q = q.filter(
func.json_contains(MarkJob.reviewer_ids,
str(current_user.user_id)))
elif current_user.user_role in [RoleEnum.annotator.value]:
q = q.filter(
func.json_contains(MarkJob.annotator_ids,
str(current_user.user_id)))
if args.get('job_id'):
q = q.filter(MarkTask.mark_job_id == args['job_id'])
if args.get('doc_type_id'):
q = q.filter(MarkJob.doc_type_id == args['doc_type_id'])
if args['task_state']:
q = q.filter(MarkTask.mark_task_status ==
status_str2int_mapper().get(args['task_state']))
if args['query']:
q = q.filter(Doc.doc_raw_name.like(f'%{args["query"]}%'))
q = q.group_by(MarkTask)
count = q.count()
processing_count = q.filter(
MarkTask.mark_task_status == int(StatusEnum.processing)).count()
if args['order_by'] and isinstance(args['order_by'], str):
if args['order_by'][1:] == 'task_id':
args['order_by'] = args['order_by'][0] + 'mark_task_id'
q = Common().order_by_model_fields(q, MarkTask, [args['order_by']])
items = []
results = q.offset(args['offset']).limit(args['limit']).all()
mark_task_ids = [mark_task.mark_task_id for mark_task, _, _ in results]
user_task_map = self._get_user_task_map(
mark_task_ids,
select_keys=(UserTask)) # .annotator_id, UserTask.mark_task_id))
UserTaskPlaceholder = UserTask(
annotator_id=0,
is_deleted=False,
user_task_status=StatusEnum.labeled.value)
for mark_task, doc_type, doc in results:
UserTaskPlaceholder.user_task_result = mark_task.mark_task_result
user_task_list = user_task_map.get(str(mark_task.mark_task_id),
[UserTaskPlaceholder])
mark_task.user_task_list = user_task_list
mark_task.doc = doc
mark_task.doc_type = doc_type
items.append(mark_task)
return count, count - processing_count, items