Beispiel #1
0
 def get(self: Resource, args: Dict[str,
                                    Any]) -> Tuple[Dict[str, Any], int]:
     """
     获取自定义容器list,分页,可以根据[custom_types]和[custom_states]过滤
     """
     # get filtered list
     filtered_list = {}
     if args.get("custom_types"):
         nlp_task_id_list = []
         for nlp_task in args.get("custom_types").split(','):
             if nlp_task == 'ner':
                 nlp_task = 'extract'
             nlp_task_id_list.append(int(NlpTaskEnum[nlp_task]))
         filtered_list.update(nlp_task_id_list=nlp_task_id_list)
     if args.get("custom_states"):
         filtered_list.update(custom_algorithm_status_list=[
             status_str2int_mapper().get(status, int(StatusEnum[status]))
             for status in args['custom_states'].split(',')
         ])
     count, custom_algorithm_list = ModelCustomService(
     ).get_custom_algorithm_list_by_filter_in(offset=args["offset"],
                                              limit=args["limit"],
                                              args=filtered_list)
     result = CustomAlgorithmSchema(many=True).dump(custom_algorithm_list)
     return {
         "message": "请求成功",
         "result": result,
         "count": count,
     }, 200
Beispiel #2
0
 def patch(self: Resource, args: typing.Dict, model_id: int,
           model_evaluate_id: int) -> typing.Tuple[typing.Dict, int]:
     """
     更新一条评估记录
     """
     update_params = {}
     if args.get("model_evaluate_state"):
         update_params.update(evaluate_task_status=status_str2int_mapper()[
             args["model_evaluate_state"]])
     if args.get("model_evaluate_result"):
         update_params.update(
             evaluate_task_result=args["model_evaluate_result"])
     if args.get("model_evaluate_name"):
         update_params.update(
             evaluate_task_name=args["model_evaluate_name"])
     if args.get("model_evaluate_desc"):
         update_params.update(
             evaluate_task_desc=args["model_evaluate_desc"])
     evaluate_task = ModelEvaluateService().update_evaluate_task_by_id(
         evaluate_task_id=model_evaluate_id, args=update_params)
     result = EvaluateTaskSchema().dump(evaluate_task)
     return {
         "message": "更新成功",
         "result": result,
     }, 200
Beispiel #3
0
 def post(self: Resource, args) -> Tuple[Dict[str, Any], int]:
     """
     更新自定义模型状态
     """
     update_params = {"custom_algorithm_status": status_str2int_mapper()[args["custom_state"]]}
     ModelCustomService().update_custom_algorithm_by_id(custom_algorithm_id=args["custom_id"], args=update_params)
     return {
                "message": "更新成功",
            }, 201
Beispiel #4
0
 def post(self: Resource,
          args: typing.Dict) -> typing.Tuple[typing.Dict, int]:
     """
     更新文档的处理状态
     """
     update_params = {"doc_status": status_str2int_mapper()[args["convert_state"]]}
     DocService().update_doc_by_id(doc_id=args["doc_id"],
                                   args=update_params)
     return {
         "message": "更新成功",
     }, 201
Beispiel #5
0
 def update_mark_task_status(current_user: CurrentUser, task_id, args):
     if args.get('task_result'):
         args['mark_task_result'] = args.get('task_result')
         del args['task_result']
     if args.get('task_state'):
         args['mark_task_status'] = status_str2int_mapper()[args['task_state']]
         del args['task_state']
     item = MarkTaskModel().update(task_id, **args)
     schema = MarkTaskSchema
     session.commit()
     result = schema().dump(item)
     return result
Beispiel #6
0
 def update_user_task_status(current_user: CurrentUser, task_id, args):
     if args.get('task_result'):
         args['user_task_result'] = args.get('task_result')
         del args['task_result']
     if args.get('task_state'):
         args['user_task_status'] = status_str2int_mapper()[args['task_state']]
         del args['task_state']
     item = UserTaskModel().update_by_annotator_id(current_user, task_id, **args)
     MarkTaskModel().check_user_task_and_update_mark_task(task_id)
     schema = UserTaskSchema
     session.commit()
     result = schema().dump(item)
     return result
Beispiel #7
0
 def put(self, args):
     """
     修改模型的状态
     """
     update_params = {}
     if args.get("model_train_state"):
         # 后端返回结果转换,失败时后端目前还返回failed
         update_params.update(train_status=status_str2int_mapper()[args["model_train_state"]])
     train_task = ModelTrainService().update_train_task_by_model_version(model_version=args["model_version"], is_check_train_terms=args["check_train_terms"], args=update_params)
     result = TrainTaskSchema().dump(train_task)
     return {
                "message": "更新成功",
                "result": result,
            }, 201
Beispiel #8
0
 def get_user_task_with_doc_and_doc_type(nlp_task_id,
                                         current_user: CurrentUser, args):
     q = session.query(UserTask, DocType, Doc) \
         .join(MarkTask, MarkTask.mark_task_id == UserTask.mark_task_id) \
         .join(MarkJob, MarkJob.mark_job_id == MarkTask.mark_job_id) \
         .join(DocType, DocType.doc_type_id == MarkJob.doc_type_id) \
         .join(Doc, Doc.doc_id == MarkTask.doc_id) \
         .filter(
         DocType.nlp_task_id == nlp_task_id,
         ~UserTask.is_deleted,
         ~MarkTask.is_deleted,
         ~Doc.is_deleted
     )
     # TODO
     # 权限
     if current_user.user_role in [
             RoleEnum.manager.value, RoleEnum.guest.value
     ]:
         q = q.filter(DocType.group_id.in_(current_user.user_groups))
     elif current_user.user_role in [RoleEnum.reviewer.value]:
         q = q.filter(
             func.json_contains(MarkJob.reviewer_ids,
                                str(current_user.user_id)))
     elif current_user.user_role in [RoleEnum.annotator.value]:
         # q = q.filter(func.json_contains(MarkJob.annotator_ids, str(current_user.user_id)))
         q = q.filter(UserTask.annotator_id == current_user.user_id)
     if args.get('job_id'):
         q = q.filter(MarkTask.mark_job_id == args['job_id'])
     if args.get('doc_type_id'):
         q = q.filter(MarkJob.doc_type_id == args['doc_type_id'])
     if args['task_state']:
         q = q.filter(MarkTask.mark_task_status ==
                      status_str2int_mapper().get(args['task_state']))
     if args['query']:
         q = q.filter(Doc.doc_raw_name.like(f'%{args["query"]}%'))
     q = q.group_by(UserTask)
     count = q.count()
     processing_count = q.filter(
         MarkTask.mark_task_status == int(StatusEnum.processing)).count()
     if args['order_by'] and isinstance(args['order_by'], str):
         if args['order_by'][1:] == 'task_id':
             args['order_by'] = args['order_by'][0] + 'mark_task_id'
         q = Common().order_by_model_fields(q, UserTask, [args['order_by']])
     items = []
     for user_task, doc_type, doc in q.offset(args['offset']).limit(
             args['limit']).all():
         user_task.doc = doc
         user_task.doc_type = doc_type
         items.append(user_task)
     return count, count - processing_count, items
Beispiel #9
0
 def put(self, args):
     """
     修改模型训练的一个字段状态
     """
     update_params = {}
     if args.get("train_term_state"):
         update_params.update(train_term_status=status_str2int_mapper()[args["train_term_state"]])
     if args.get("train_term_result"):
         update_params.update(train_term_result=args["train_term_result"])
     train_term_task = ModelTrainService().update_train_term_by_model_version_and_doc_term_id(model_version=args["model_version"],
                                                                                              doc_term_id=args["doc_term_id"],
                                                                                              args=update_params)
     result = TrainTermTaskSchema().dump(train_term_task)
     return {
                "message": "更新成功",
                "result": result,
            }, 201
Beispiel #10
0
    def post(self: Resource, args: typing.Dict) -> typing.Tuple[typing.Dict, int]:
        """
        message queue回调统一入口
        """
        message = args['message']
        logger.info(f"receive callback info from mq. response is: {json.dumps(args)}")

        if message['business'] in [
            'label',  # 实体预标注
            'classify_label',  # 分类预标注
            'relation_label',  # 实体关系预标注
            'wordseg_label'  # 分词预标注
        ]:
            update_params = {}
            if args.get("task_state"):
                if args['task_state'] == 'success':     # 如果mq预标注返回成功,则初试状态是unlabel
                    update_params.update(mark_task_status=int(StatusEnum.unlabel))
                else:   # 如果mq预标注返回失败,则初试状态是fail
                    update_params.update(mark_task_status=int(StatusEnum.fail))
            if args.get("task_result"):
                update_params.update(mark_task_result=args["task_result"])
            mark_task, user_task_list = MarkJobService()\
                .update_mark_task_and_user_task_by_mark_task_id(mark_task_id=message["task_id"], args=update_params)
            MarkJobService().update_mark_job_status_by_mark_task(mark_task=mark_task)
            result = UserTaskSchema(many=True).dump(user_task_list)
            return {
                       "message": "更新成功",
                       "result": result,
                   }, 201
        elif message['business'] in [
            'extract',  # 实体抽取
            'classify_extract',  # 分类抽取
            'relation_extract',  # 实体关系抽取
            'wordseg_extract'  # 分词抽取
        ]:
            update_params = {}
            if args.get("task_state"):
                update_params.update(predict_task_status=status_str2int_mapper()[args["task_state"]])
            if args.get("task_result"):
                update_params.update(predict_task_result=args["task_result"])
            predict_task = PredictService().update_predict_task_by_id(predict_task_id=message["task_id"], args=update_params)
            result = PredictTaskSchema().dump(predict_task)
            return {
                       "message": "更新成功",
                       "result": result,
                   }, 201
Beispiel #11
0
 def patch(self: Resource, args: typing.Dict,
           job_id: int) -> typing.Tuple[typing.Dict, int]:
     update_params = {}
     if args.get("extract_job_name"):
         update_params.update(predict_job_name=args["extract_job_name"])
     if args.get("extract_job_desc"):
         update_params.update(predict_job_desc=args["extract_job_desc"])
     if args.get("extract_job_state"):
         update_params.update(predict_job_status=status_str2int_mapper()[
             args["extract_job_state"]])
     predict_job = PredictService().update_predict_job_by_id(
         predict_job_id=job_id, args=update_params)
     result = PredictJobSchema().dump(predict_job)
     return {
         "message": "更新成功",
         "result": result,
     }, 200
Beispiel #12
0
 def patch(self: Resource, args: typing.Dict,
           task_id: int) -> typing.Tuple[typing.Dict, int]:
     update_params = {}
     if args.get("task_state"):
         update_params.update(predict_task_status=status_str2int_mapper()[
             args["task_state"]])
     if args.get("task_result"):
         update_params.update(predict_task_result=args["task_result"])
     predict_task = TaskMachineService().update_predict_task_by_id(
         predict_task_id=task_id, args=update_params)
     result = PredictTaskSchema().dump(predict_task)
     # 根据task更新整个predict_job的状态
     TaskMachineService().update_predict_job_status_by_predict_task(
         predict_task=predict_task)
     return {
         "message": "更新成功",
         "result": result,
     }, 200
    def put(self, args):
        """
        更新一条评估记录
        """
        update_params = {}
        if args.get("model_evaluate_state"):
            update_params.update(evaluate_task_status=status_str2int_mapper()[
                args["model_evaluate_state"]])
        if args.get("model_evaluate_result"):
            update_params.update(
                evaluate_task_result=args["model_evaluate_result"])

        evaluate_task = ModelEvaluateService().update_evaluate_task_by_id(
            evaluate_task_id=args["model_evaluate_id"], args=update_params)
        result = EvaluateTaskSchema().dump(evaluate_task)
        return {
            "message": "更新成功",
            "result": result,
        }, 201
Beispiel #14
0
 def patch(
         self: Resource,
         args: typing.Dict,
         model_id: int,
         model_train_id: int
 ) -> typing.Tuple[typing.Dict, int]:
     """
     修改模型的状态和结果
     """
     update_params = {}
     if args.get("model_train_state"): # 这里不考虑model_train_result因为新的表结构里没有这个列了
         update_params.update(train_status=status_str2int_mapper()[args["model_train_state"]])
     train_task = ModelTrainService().update_train_task_by_id(train_job_id=model_id, train_task_id=model_train_id,
                                                              is_check_train_terms=args["check_train_terms"], model_type = args["model_type"],
                                                              args=update_params)
     result = TrainTaskSchema().dump(train_task)
     return {
                "message": "更新成功",
                "result": result,
            }, 200
Beispiel #15
0
 def patch(
         self: Resource,
         args: typing.Dict,
         model_id: int,
         model_train_id: int,
         train_term_id: int,
 ) -> typing.Tuple[typing.Dict, int]:
     """
     修改模型训练的一个字段状态
     """
     update_params = {}
     if args.get("train_term_state"):
         update_params.update(train_term_status=status_str2int_mapper()[args["train_term_state"]])
     if args.get("train_term_result"):
         update_params.update(train_term_result=args["train_term_result"])
     train_term_task = ModelTrainService().update_train_task_term_by_id(train_term_task_id=train_term_id, args=update_params)
     result = TrainTermTaskSchema().dump(train_term_task)
     return {
                "message": "更新成功",
                "result": result,
            }, 200
Beispiel #16
0
 def get(self: Resource, args: typing.Dict):
     order_by = args["order_by"][1:]
     order_by_desc = True if args["order_by"][0] == "-" else False
     filtered_list = {}
     if args.get("task_state"):
         filtered_list.update(predict_task_status=status_str2int_mapper()[
             args["task_state"]])
     count, predict_task_list = TaskMachineService(
     ).get_predict_task_list_by_predict_job_id(
         predict_job_id=args["extract_job_id"],
         search=args['query'],
         order_by=order_by,
         order_by_desc=order_by_desc,
         offset=args['offset'],
         limit=args['limit'],
         current_user=self.get_current_user(),
         args=filtered_list)
     result = PredictTaskSchema(
         many=True, exclude=('task_result', )).dump(predict_task_list)
     return {
         "message": "请求成功",
         "result": result,
         "count": count,
     }, 200
Beispiel #17
0
 def get_mark_task_with_doc_and_doc_type(self, nlp_task_id,
                                         current_user: CurrentUser, args):
     q = session.query(MarkTask, DocType, Doc) \
         .join(MarkJob, MarkJob.mark_job_id == MarkTask.mark_job_id) \
         .join(DocType, DocType.doc_type_id == MarkJob.doc_type_id) \
         .join(Doc, Doc.doc_id == MarkTask.doc_id) \
         .filter(
         DocType.nlp_task_id == nlp_task_id,
         ~DocType.is_deleted,
         ~MarkTask.is_deleted,
         ~Doc.is_deleted
     )
     # TODO
     # 权限
     if current_user.user_role in [
             RoleEnum.manager.value, RoleEnum.guest.value
     ]:
         q = q.filter(DocType.group_id.in_(current_user.user_groups))
     elif current_user.user_role in [RoleEnum.reviewer.value]:
         q = q.filter(
             func.json_contains(MarkJob.reviewer_ids,
                                str(current_user.user_id)))
     elif current_user.user_role in [RoleEnum.annotator.value]:
         q = q.filter(
             func.json_contains(MarkJob.annotator_ids,
                                str(current_user.user_id)))
     if args.get('job_id'):
         q = q.filter(MarkTask.mark_job_id == args['job_id'])
     if args.get('doc_type_id'):
         q = q.filter(MarkJob.doc_type_id == args['doc_type_id'])
     if args['task_state']:
         q = q.filter(MarkTask.mark_task_status ==
                      status_str2int_mapper().get(args['task_state']))
     if args['query']:
         q = q.filter(Doc.doc_raw_name.like(f'%{args["query"]}%'))
     q = q.group_by(MarkTask)
     count = q.count()
     processing_count = q.filter(
         MarkTask.mark_task_status == int(StatusEnum.processing)).count()
     if args['order_by'] and isinstance(args['order_by'], str):
         if args['order_by'][1:] == 'task_id':
             args['order_by'] = args['order_by'][0] + 'mark_task_id'
         q = Common().order_by_model_fields(q, MarkTask, [args['order_by']])
     items = []
     results = q.offset(args['offset']).limit(args['limit']).all()
     mark_task_ids = [mark_task.mark_task_id for mark_task, _, _ in results]
     user_task_map = self._get_user_task_map(
         mark_task_ids,
         select_keys=(UserTask))  # .annotator_id, UserTask.mark_task_id))
     UserTaskPlaceholder = UserTask(
         annotator_id=0,
         is_deleted=False,
         user_task_status=StatusEnum.labeled.value)
     for mark_task, doc_type, doc in results:
         UserTaskPlaceholder.user_task_result = mark_task.mark_task_result
         user_task_list = user_task_map.get(str(mark_task.mark_task_id),
                                            [UserTaskPlaceholder])
         mark_task.user_task_list = user_task_list
         mark_task.doc = doc
         mark_task.doc_type = doc_type
         items.append(mark_task)
     return count, count - processing_count, items