class UpdateModelEvaluateResource(Resource):
@parse({
"model_evaluate_id":
fields.Integer(required=True),
"model_evaluate_state":
fields.String(required=True),
"model_evaluate_result":
fields.Dict(),
"model_type":
fields.String(required=True,
validate=lambda x: x in
('extract', 'classify', 'relation', 'wordseg'))
})
def put(self, args):
"""
更新一条评估记录
"""
update_params = {}
if args.get("model_evaluate_state"):
update_params.update(evaluate_task_status=status_str2int_mapper()[
args["model_evaluate_state"]])
if args.get("model_evaluate_result"):
update_params.update(
evaluate_task_result=args["model_evaluate_result"])
evaluate_task = ModelEvaluateService().update_evaluate_task_by_id(
evaluate_task_id=args["model_evaluate_id"], args=update_params)
result = EvaluateTaskSchema().dump(evaluate_task)
return {
"message": "更新成功",
"result": result,
}, 201
class TrainTermItemResource(Resource):
@parse({
"train_term_state": fields.String(),
"train_term_result": fields.Dict(),
"model_type": fields.String(required=True, validate=lambda x: x in ('extract', 'classify')),
})
def patch(
self: Resource,
args: typing.Dict,
model_id: int,
model_train_id: int,
train_term_id: int,
) -> typing.Tuple[typing.Dict, int]:
"""
修改模型训练的一个字段状态
"""
update_params = {}
if args.get("train_term_state"):
update_params.update(train_term_status=status_str2int_mapper()[args["train_term_state"]])
if args.get("train_term_result"):
update_params.update(train_term_result=args["train_term_result"])
train_term_task = ModelTrainService().update_train_task_term_by_id(train_term_task_id=train_term_id, args=update_params)
result = TrainTermTaskSchema().dump(train_term_task)
return {
"message": "更新成功",
"result": result,
}, 200
class UpdateTrainTermResource(Resource):
@parse({
"model_version": fields.String(required=True),
"doc_term_id": fields.Integer(required=True),
"train_term_state": fields.String(required=True),
"train_term_result": fields.Dict(),
"term_type": fields.String(required=True),
"model_type": fields.String(required=True,
validate=lambda x: x in ('extract', 'classify', 'relation', 'wordseg'))
})
def put(self, args):
"""
修改模型训练的一个字段状态
"""
update_params = {}
if args.get("train_term_state"):
update_params.update(train_term_status=status_str2int_mapper()[args["train_term_state"]])
if args.get("train_term_result"):
update_params.update(train_term_result=args["train_term_result"])
train_term_task = ModelTrainService().update_train_term_by_model_version_and_doc_term_id(model_version=args["model_version"],
doc_term_id=args["doc_term_id"],
args=update_params)
result = TrainTermTaskSchema().dump(train_term_task)
return {
"message": "更新成功",
"result": result,
}, 201
class ClassifyDocRuleItemResource(Resource):
def get(self, doc_type_id, doc_rule_id):
result = DocTermService().get_classify_rule(doc_rule_id)
return {
"message": "请求成功",
"result": result,
}, 200
@parse({
"rule_content": fields.Dict(),
"state": fields.Integer()
})
def patch(self, args, doc_type_id, doc_rule_id):
result = DocTermService().update_classify_rule(args, doc_rule_id)
DocTermService().update_rule_to_redis(doc_type_id)
r.delete(f'classify:rule:{doc_type_id}')
return {
"message": "更新成功",
"result": result,
}, 200
def delete(self, doc_type_id, doc_rule_id):
DocTermService().delete_doc_rule(doc_rule_id)
DocTermService().update_rule_to_redis(doc_type_id)
r.delete(f'classify:rule:{doc_type_id}')
return {
"message": "删除成功",
}, 204
class TrainTermTaskSchema(Schema): # type: ignore
train_term_id = fields.Integer(attribute="train_term_task_id")
train_term_result = fields.Dict()
train_term_state = fields.Function(
lambda obj: StatusEnum(obj.train_term_status).name)
model_train_id = fields.Integer(attribute="train_task_id")
doc_term_id = fields.Integer()
class PredictTaskSchema(Schema):
doc_type = fields.Nested(DocTypeSchema)
doc = fields.Nested(DocSchema)
task_id = fields.Integer(attribute="predict_task_id")
task_result = fields.Dict(attribute="predict_task_result")
task_state = fields.Function(
lambda obj: StatusEnum(obj.predict_task_status).name)
created_time = fields.DateTime()
class TrainTaskSchema(Schema): # type: ignore
model_train_id = fields.Integer(attribute="train_task_id")
model_train_config = fields.Dict(attribute="train_config")
model_train_state = fields.Function(
lambda obj: StatusEnum(obj.train_status).name)
model_id = fields.Integer(attribute="train_job_id")
mark_job_ids = fields.List(fields.Integer())
train_terms = fields.List(fields.Nested(TrainTermTaskSchema))
created_time = fields.DateTime()
last_updated_time = fields.DateTime(attribute="updated_time")
model_version = fields.String()
class EvaluateTaskSchema(Schema):
model_evaluate_id = fields.Integer(attribute="evaluate_task_id")
model_evaluate_name = fields.String(attribute="evaluate_task_name")
model_evaluate_desc = fields.String(attribute="evaluate_task_desc")
model_evaluate_state = fields.Function(
lambda obj: StatusEnum(obj.evaluate_task_status).name)
model_evaluate_result = fields.Dict(attribute="evaluate_task_result")
model_id = fields.Integer(attribute="train_job_id")
mark_job_ids = fields.List(fields.Integer())
created_time = fields.DateTime()
last_updated_time = fields.DateTime(attribute="updated_time")
class TrainJobSchema(Schema): # type: ignore
model_id = fields.Integer(attribute='train_job_id')
model_name = fields.String(attribute="train_job_name")
model_desc = fields.String(attribute="train_job_desc")
status = fields.Function(lambda obj: not obj.is_deleted)
doc_type = fields.Nested(DocTypeSchema)
created_time = fields.String()
model_version = fields.String()
train_list = fields.List(fields.Nested(TrainTaskSchema))
model_evaluate = fields.Nested(EvaluateTaskSchema)
preprocess = fields.Dict()
class ClassifyDocRuleListResource(Resource):
@parse({
"offset": fields.Integer(missing=0),
"limit": fields.Integer(missing=10),
"timestamp": fields.String(),
})
def get(self, args, doc_type_id):
"""
规则列表
"""
redis_key = f'classify:rule:{doc_type_id}'
if args.get("timestamp"):
try:
result = json.loads(r.get(redis_key))
if result['timestamp'] == args["timestamp"]:
result.update(update=False)
return result
except Exception:
pass
# TODO 查询优化
result, count, timestamp = DocTermService().get_classify_doc_rule(doc_type_id, args.get("offset"),
args.get("limit"))
data = {
"message": "请求成功",
"result": result,
"timestamp": timestamp,
"update": True,
"count": count,
}
if args.get("timestamp"):
r.set(redis_key, json.dumps(data), ex=24 * 60 * 60)
return data, 200
@parse({
"doc_term_id": fields.Integer(required=True),
"rule_type": fields.String(required=True),
"rule_content": fields.Dict(required=True)
})
def post(self, args, doc_type_id):
try:
result = DocTermService().create_new_rule(args)
DocTermService().update_rule_to_redis(doc_type_id)
r.delete(f'classify:rule:{doc_type_id}')
return {
"message": "创建成功",
"result": result,
}, 201
except ValueError as e:
abort(400, message=str(e))
class ClassifyModelListResource(Resource, CurrentUserMixin):
@parse({
"query": fields.String(missing=''),
"offset": fields.Integer(missing=0),
"limit": fields.Integer(missing=10),
"doc_type_id": fields.Integer(missing=0),
'order_by': fields.String(missing='-created_time'),
})
def get(self: Resource, args: Dict[str,
Any]) -> Tuple[Dict[str, Any], int]:
"""
获取模型记录，分页
"""
count, train_job_list = ModelService(
).get_train_job_list_by_nlp_task_id(
nlp_task_id=int(NlpTaskEnum.classify),
doc_type_id=args['doc_type_id'],
search=args['query'],
offset=args['offset'],
limit=args['limit'],
current_user=self.get_current_user())
# get the serialized result
result = TrainJobSchema().dump(train_job_list, many=True)
return {
"message": "请求成功",
"result": result,
"count": count,
}, 200
@parse({
"model_name": fields.String(required=True),
"model_desc": fields.String(missing=""),
"doc_type_id": fields.Integer(required=True),
"model_train_config": fields.Dict(required=True),
"mark_job_ids": fields.List(fields.Integer(), missing=[]),
"custom_id": fields.Integer(missing=0),
})
def post(self: Resource, args: Dict[str,
Any]) -> Tuple[Dict[str, Any], int]:
train_job = ModelService().create_classify_train_job_by_doc_type_id(
doc_type_id=args["doc_type_id"],
train_job_name=args["model_name"],
train_job_desc=args["model_desc"],
train_config=args["model_train_config"],
mark_job_ids=args["mark_job_ids"],
custom_id=args['custom_id'])
result = TrainJobSchema().dump(train_job)
return {"message": "创建成功", "result": result}, 201
class CustomAlgorithmSchema(Schema):
custom_id = fields.Integer(attribute="custom_algorithm_id")
custom_ip = fields.String(attribute="custom_algorithm_ip")
custom_port = fields.Integer(attribute="custom_algorithm_predict_port")
custom_evaluate_port = fields.Integer(
attribute="custom_algorithm_evaluate_port")
custom_name = fields.String(attribute="custom_algorithm_name")
custom_id_name = fields.String(attribute="custom_algorithm_alias")
custom_desc = fields.String(attribute="custom_algorithm_desc")
custom_type = fields.Function(
lambda obj: "ner"
if obj.nlp_task_id == NlpTaskEnum.extract and obj.preprocess.get(
"split_by_sentence", False) else NlpTaskEnum(obj.nlp_task_id).name)
custom_state = fields.Function(
lambda obj: StatusEnum(obj.custom_algorithm_status).name)
custom_config = fields.String(attribute="custom_algorithm_config")
created_time = fields.DateTime()
preprocess = fields.Dict()
class UpdateModelTrainResource(Resource):
@parse({
"model_version": fields.String(required=True),
"check_train_terms": fields.Boolean(required=True),
"model_train_state": fields.String(),
"model_train_result": fields.Dict(),
"model_type": fields.String(required=True,
validate=lambda x: x in ('extract', 'classify', 'relation', 'wordseg'))
})
def put(self, args):
"""
修改模型的状态
"""
update_params = {}
if args.get("model_train_state"):
# 后端返回结果转换，失败时后端目前还返回failed
update_params.update(train_status=status_str2int_mapper()[args["model_train_state"]])
train_task = ModelTrainService().update_train_task_by_model_version(model_version=args["model_version"], is_check_train_terms=args["check_train_terms"], args=update_params)
result = TrainTaskSchema().dump(train_task)
return {
"message": "更新成功",
"result": result,
}, 201
class ClassifyDocRuleSchema(Schema): # type: ignore
id = fields.Integer(attribute="classify_rule_id")
rule_type = fields.String()
rule_content = fields.Dict()
doc_term_id = fields.Integer()
state = fields.Integer(attribute="is_active")
class ModelTrainItemResource(Resource):
@parse({
"model_type": fields.String(required=True, validate=lambda x: x in ("extract", "classify", "relation", "wordseg"))
})
def get(
self: Resource,
args: typing.Dict,
model_id: int,
model_train_id: int
) -> typing.Tuple[typing.Dict, int]:
"""
获取单条模型训练记录
"""
# get single task task by id
train_task = ModelTrainService().get_train_task_by_id(train_task_id=model_train_id)
result = TrainTaskSchema().dump(train_task)
# add extra algorithm information for extract and relation.
if args["model_type"] in ["extract", "relation"]:
result = ModelTrainService().add_algo_dict_for_extract_relation(result)
return {
"message": "请求成功",
"result": result,
}, 200
@parse({
"model_train_state": fields.String(),
"model_train_result": fields.Dict(),
"check_train_terms": fields.Boolean(missing=False),
"model_type": fields.String(required=True, validate=lambda x: x in ("extract", "classify", "relation", "wordseg"))
})
def patch(
self: Resource,
args: typing.Dict,
model_id: int,
model_train_id: int
) -> typing.Tuple[typing.Dict, int]:
"""
修改模型的状态和结果
"""
update_params = {}
if args.get("model_train_state"): # 这里不考虑model_train_result因为新的表结构里没有这个列了
update_params.update(train_status=status_str2int_mapper()[args["model_train_state"]])
train_task = ModelTrainService().update_train_task_by_id(train_job_id=model_id, train_task_id=model_train_id,
is_check_train_terms=args["check_train_terms"], model_type = args["model_type"],
args=update_params)
result = TrainTaskSchema().dump(train_task)
return {
"message": "更新成功",
"result": result,
}, 200
@parse({
"model_type": fields.String(required=True, validate=lambda x: x in ("extract", "classify"))
})
def delete(
self: Resource,
args: typing.Dict,
model_id: int,
model_train_id: int
) -> typing.Tuple[typing.Dict, int]:
"""
删除模型
"""
ModelTrainService().delete_train_task_by_id(train_task_id=model_train_id)
return {
"message": "删除成功",
}, 200
class ModelEvaluateItemResource(Resource):
@parse({
"model_type":
fields.String(required=True,
validate=lambda x: x in
('extract', 'classify', 'relation', 'wordseg'))
})
def get(self: Resource, args: typing.Dict, model_id: int,
model_evaluate_id: int) -> typing.Tuple[typing.Dict, int]:
"""
获取单条模型评估记录
"""
evaluate_task = ModelEvaluateService().get_evaluate_task_by_id(
model_evaluate_id)
result = EvaluateTaskSchema().dump(evaluate_task)
return {
"message": "请求成功",
"result": result,
}, 200
@parse({
"model_evaluate_name":
fields.String(),
"model_evaluate_desc":
fields.String(),
"model_evaluate_state":
fields.String(),
"model_evaluate_result":
fields.Dict(),
"model_type":
fields.String(required=True,
validate=lambda x: x in
('extract', 'classify', 'relation', 'wordseg'))
})
def patch(self: Resource, args: typing.Dict, model_id: int,
model_evaluate_id: int) -> typing.Tuple[typing.Dict, int]:
"""
更新一条评估记录
"""
update_params = {}
if args.get("model_evaluate_state"):
update_params.update(evaluate_task_status=status_str2int_mapper()[
args["model_evaluate_state"]])
if args.get("model_evaluate_result"):
update_params.update(
evaluate_task_result=args["model_evaluate_result"])
if args.get("model_evaluate_name"):
update_params.update(
evaluate_task_name=args["model_evaluate_name"])
if args.get("model_evaluate_desc"):
update_params.update(
evaluate_task_desc=args["model_evaluate_desc"])
evaluate_task = ModelEvaluateService().update_evaluate_task_by_id(
evaluate_task_id=model_evaluate_id, args=update_params)
result = EvaluateTaskSchema().dump(evaluate_task)
return {
"message": "更新成功",
"result": result,
}, 200
@parse({
"model_type":
fields.String(required=True,
validate=lambda x: x in
('extract', 'classify', 'relation', 'wordseg'))
})
def delete(self: Resource, args: typing.Dict, model_id: int,
model_evaluate_id: int) -> typing.Tuple[typing.Dict, int]:
"""
删除一条评估记录
"""
ModelEvaluateService().delete_evaluate_task_by_id(
evaluate_task_id=model_evaluate_id)
return {
"message": "删除成功",
}, 200