def save_machine_learning_model_info(self):
try:
record = MLModel.get_or_none(
MLModel.f_model_version == self.job_id)
if not record:
job = Job.get_or_none(Job.f_job_id == self.job_id)
if job:
job_data = job.to_json()
MLModel.create(
f_role=self.role,
f_party_id=self.party_id,
f_roles=job_data.get("f_roles"),
f_model_id=self.model_id,
f_model_version=self.model_version,
f_job_id=job_data.get("f_job_id"),
f_create_time=current_timestamp(),
f_initiator_role=job_data.get('f_initiator_role'),
f_initiator_party_id=job_data.get(
'f_initiator_party_id'),
f_runtime_conf=job_data.get('f_runtime_conf'),
f_work_mode=job_data.get('f_work_mode'),
f_dsl=job_data.get('f_dsl'),
f_train_runtime_conf=job_data.get(
'f_train_runtime_conf'),
f_size=self.get_model_size(),
f_job_status=job_data.get('f_status'))
schedule_logger(self.job_id).info(
'save {} model info done. model id: {}, model version: {}.'
.format(self.job_id, self.model_id,
self.model_version))
else:
schedule_logger(self.job_id).info(
'save {} model info failed, no job found in db. '
'model id: {}, model version: {}.'.format(
self.job_id, self.model_id, self.model_version))
else:
schedule_logger(self.job_id).info(
'model {} info has already existed in database.'.format(
self.job_id))
except Exception as e:
schedule_logger(self.job_id).exception(e)
def operate_model(model_operation):
request_config = request.json or request.form.to_dict()
job_id = job_utils.generate_job_id()
if model_operation not in [
ModelOperation.STORE, ModelOperation.RESTORE,
ModelOperation.EXPORT, ModelOperation.IMPORT
]:
raise Exception(
'Can not support this operating now: {}'.format(model_operation))
required_arguments = ["model_id", "model_version", "role", "party_id"]
check_config(request_config, required_arguments=required_arguments)
request_config["model_id"] = gen_party_model_id(
model_id=request_config["model_id"],
role=request_config["role"],
party_id=request_config["party_id"])
if model_operation in [ModelOperation.EXPORT, ModelOperation.IMPORT]:
if model_operation == ModelOperation.IMPORT:
try:
file = request.files.get('file')
file_path = os.path.join(TEMP_DIRECTORY, file.filename)
# if not os.path.exists(file_path):
# raise Exception('The file is obtained from the fate flow client machine, but it does not exist, '
# 'please check the path: {}'.format(file_path))
try:
os.makedirs(os.path.dirname(file_path), exist_ok=True)
file.save(file_path)
except Exception as e:
shutil.rmtree(file_path)
raise e
request_config['file'] = file_path
model = pipelined_model.PipelinedModel(
model_id=request_config["model_id"],
model_version=request_config["model_version"])
model.unpack_model(file_path)
pipeline = model.read_component_model('pipeline',
'pipeline')['Pipeline']
train_runtime_conf = json_loads(pipeline.train_runtime_conf)
permitted_party_id = []
for key, value in train_runtime_conf.get('role', {}).items():
for v in value:
permitted_party_id.extend([v, str(v)])
if request_config["party_id"] not in permitted_party_id:
shutil.rmtree(model.model_path)
raise Exception(
"party id {} is not in model roles, please check if the party id is valid."
)
try:
from fate_flow.utils.config_adapter import JobRuntimeConfigAdapter
adapter = JobRuntimeConfigAdapter(train_runtime_conf)
job_parameters = adapter.get_common_parameters().to_dict()
with DB.connection_context():
model = MLModel.get_or_none(
MLModel.f_job_id == job_parameters.get(
"model_version"),
MLModel.f_role == request_config["role"])
if not model:
MLModel.create(
f_role=request_config["role"],
f_party_id=request_config["party_id"],
f_roles=train_runtime_conf["role"],
f_job_id=train_runtime_conf["job_parameters"]
["model_version"],
f_model_id=train_runtime_conf["job_parameters"]
["model_id"],
f_model_version=train_runtime_conf[
"job_parameters"]["model_version"],
f_initiator_role=train_runtime_conf[
"initiator"]["role"],
f_initiator_party_id=train_runtime_conf[
"initiator"]["party_id"],
f_runtime_conf=train_runtime_conf,
f_work_mode=train_runtime_conf[
"job_parameters"]["work_mode"],
f_dsl=json_loads(pipeline.train_dsl),
f_imported=1,
f_job_status=JobStatus.SUCCESS)
else:
stat_logger.info(
f'job id: {job_parameters.get("model_version")}, '
f'role: {request_config["role"]} model info already existed in database.'
)
except peewee.IntegrityError as e:
stat_logger.exception(e)
operation_record(request_config, "import", "success")
return get_json_result()
except Exception:
operation_record(request_config, "import", "failed")
raise
else:
try:
model = pipelined_model.PipelinedModel(
model_id=request_config["model_id"],
model_version=request_config["model_version"])
if model.exists():
archive_file_path = model.packaging_model()
operation_record(request_config, "export", "success")
return send_file(archive_file_path,
attachment_filename=os.path.basename(
archive_file_path),
as_attachment=True)
else:
operation_record(request_config, "export", "failed")
res = error_response(
response_code=210,
retmsg="Model {} {} is not exist.".format(
request_config.get("model_id"),
request_config.get("model_version")))
return res
except Exception as e:
operation_record(request_config, "export", "failed")
stat_logger.exception(e)
return error_response(response_code=210, retmsg=str(e))
else:
data = {}
job_dsl, job_runtime_conf = gen_model_operation_job_config(
request_config, model_operation)
submit_result = DAGScheduler.submit(
{
'job_dsl': job_dsl,
'job_runtime_conf': job_runtime_conf
},
job_id=job_id)
data.update(submit_result)
operation_record(data=job_runtime_conf,
oper_type=model_operation,
oper_status='')
return get_json_result(job_id=job_id, data=data)