def restore(self, model_id: str, model_version: str, store_address: dict):
"""
Restore model from cos to local cache
:param model_id:
:param model_version:
:param store_address:
:return:
"""
store_key = self.store_key(model_id, model_version) + '.zip'
model = PipelinedModel(model_id, model_version)
cos = self.get_connection(store_address)
try:
cos.download_file(
Bucket=store_address["Bucket"],
Key=store_key,
DestFilePath=model.archive_model_file_path,
EnableCRC=True,
)
model.unpack_model(model.archive_model_file_path)
except Exception as e:
LOGGER.exception(e)
raise Exception(
f"Restore model {model_id} {model_version} from Tencent COS failed."
)
else:
LOGGER.info(
f"Restore model {model_id} {model_version} from Tencent COS successfully. "
f"Archive path: {model.archive_model_file_path} Key: {store_key}"
)
def restore(self, model_id: str, model_version: str, store_address: dict):
"""
Restore model from redis to local cache
:param model_id:
:param model_version:
:param store_address:
:return:
"""
store_key = self.store_key(model_id, model_version)
model = PipelinedModel(model_id, model_version)
red = self.get_connection(store_address)
try:
archive_data = red.get(name=store_key)
if not archive_data:
raise TypeError(
f"The key {store_key} does not exists or is empty.")
with open(model.archive_model_file_path, "wb") as fw:
fw.write(archive_data)
model.unpack_model(model.archive_model_file_path)
except Exception as e:
LOGGER.exception(e)
raise Exception(
f"Restore model {model_id} {model_version} from redis failed.")
else:
LOGGER.info(
f"Restore model {model_id} {model_version} from redis successfully. "
f"Archive path: {model.archive_model_file_path} Key: {store_key}"
)
def restore(self, model_id: str, model_version: str, store_address: dict):
"""
Restore model from redis to local cache
:param model_id:
:param model_version:
:param store_address:
:return:
"""
try:
red = self.get_connection(config=store_address)
model = PipelinedModel(model_id=model_id,
model_version=model_version)
redis_store_key = self.store_key(model_id=model_id,
model_version=model_version)
model_archive_data = red.get(name=redis_store_key)
if not model_archive_data:
raise Exception(
"Restore model {} {} to redis failed: {}".format(
model_id, model_version,
"can not found model archive data"))
with open(model.archive_model_file_path, "wb") as fw:
fw.write(model_archive_data)
model.unpack_model(model.archive_model_file_path)
LOGGER.info(
"Restore model to {} from redis successfully using key {}".
format(model.archive_model_file_path, redis_store_key))
except Exception as e:
LOGGER.exception(e)
raise Exception("Restore model {} {} from redis failed".format(
model_id, model_version))
def store(self,
model_id: str,
model_version: str,
store_address: dict,
force_update: bool = False):
"""
Store the model from local cache to redis
:param model_id:
:param model_version:
:param store_address:
:param force_update:
:return:
"""
try:
red = self.get_connection(config=store_address)
model = PipelinedModel(model_id=model_id,
model_version=model_version)
redis_store_key = self.store_key(model_id=model_id,
model_version=model_version)
with open(model.packaging_model(), "rb") as fr:
red.set(name=redis_store_key,
value=fr.read(),
ex=store_address.get("ex", None),
nx=True if not force_update else False)
LOGGER.info(
"Store model {} {} to redis successfully using key {}".format(
model_id, model_version, redis_store_key))
except Exception as e:
LOGGER.exception(e)
raise Exception("Store model {} {} to redis failed".format(
model_id, model_version))
def query_model_info_from_file(model_id=None,
model_version=None,
role=None,
party_id=None,
query_filters=None,
to_dict=False,
**kwargs):
res = {} if to_dict else []
model_dir = os.path.join(get_fate_flow_directory(), 'model_local_cache')
glob_dir = f"{model_dir}{os.sep}{role if role else '*'}#{party_id if party_id else '*'}#{model_id if model_id else '*'}{os.sep}{model_version if model_version else '*'}"
stat_logger.info(f'glob model dir: {glob_dir}')
model_fp_list = glob.glob(glob_dir)
if model_fp_list:
for fp in model_fp_list:
pipeline_model = PipelinedModel(model_id=fp.split(os.path.sep)[-2],
model_version=fp.split(
os.path.sep)[-1])
model_info = gather_model_info_data(pipeline_model,
query_filters=query_filters)
if model_info:
_role = fp.split('/')[-2].split('#')[0]
_party_id = fp.split('/')[-2].split('#')[1]
model_info["f_role"] = _role
model_info["f_party_id"] = _party_id
if isinstance(res, dict):
res[fp] = model_info
else:
res.append(model_info)
if kwargs.get('save'):
try:
insert_info = gather_model_info_data(
pipeline_model).copy()
insert_info['role'] = _role
insert_info['party_id'] = _party_id
insert_info['job_id'] = insert_info.get(
'f_model_version')
insert_info[
'size'] = pipeline_model.calculate_model_file_size(
)
if compare_version(insert_info['f_fate_version'],
'1.5.1') == 'lt':
insert_info['roles'] = insert_info.get(
'f_train_runtime_conf', {}).get('role', {})
insert_info['initiator_role'] = insert_info.get(
'f_train_runtime_conf',
{}).get('initiator', {}).get('role')
insert_info[
'initiator_party_id'] = insert_info.get(
'f_train_runtime_conf',
{}).get('initiator', {}).get('party_id')
save_model_info(insert_info)
except Exception as e:
stat_logger.exception(e)
if res:
return 0, 'Query model info from local model success.', res
return 100, 'Query model info failed, cannot find model from local model files.', res
def setUp(self):
shutil.rmtree(TEMP_DIRECTORY, True)
self.pipelined_model = PipelinedModel('foobar', 'v1')
shutil.rmtree(self.pipelined_model.model_path, True)
self.pipelined_model.create_pipelined_model()
with open(self.pipelined_model.define_meta_path, 'w',
encoding='utf8') as f:
yaml.dump(data_define_meta, f)
def check_if_deployed(role, party_id, model_id, model_version):
party_model_id = gen_party_model_id(model_id=model_id,
role=role,
party_id=party_id)
pipeline_model = PipelinedModel(model_id=party_model_id,
model_version=model_version)
if not pipeline_model.exists():
raise Exception(
f"Model {party_model_id} {model_version} not exists in model local cache."
)
pipeline = pipeline_model.read_pipeline_model()
if compare_version(pipeline.fate_version, '1.5.0') == 'gt':
train_runtime_conf = json_loads(pipeline.train_runtime_conf)
if str(train_runtime_conf.get('dsl_version', '1')) != '1':
if pipeline.parent:
return False
return True
def check_before_deploy(pipeline_model: PipelinedModel):
pipeline = pipeline_model.read_pipeline_model()
if compare_version(pipeline.fate_version, '1.5.0') == 'gt':
if pipeline.parent:
return True
elif compare_version(pipeline.fate_version, '1.5.0') == 'eq':
return True
return False
def store(self,
model_id: str,
model_version: str,
store_address: dict,
force_update: bool = False):
"""
Store the model from local cache to redis
:param model_id:
:param model_version:
:param store_address:
:param force_update:
:return:
"""
store_key = self.store_key(model_id, model_version)
if not force_update and self.exists(model_id, model_version,
store_address):
raise FileExistsError(f"The key {store_key} already exists.")
model = PipelinedModel(model_id, model_version)
red = self.get_connection(store_address)
try:
model.packaging_model()
with open(model.archive_model_file_path, "rb") as fr:
res = red.set(store_key,
fr.read(),
nx=not force_update,
ex=store_address.get("ex", None))
if res is not True:
if not force_update:
raise FileExistsError(
f"The key {store_key} already exists.")
raise TypeError(f"Execute command failed.")
except Exception as e:
LOGGER.exception(e)
raise Exception(
f"Store model {model_id} {model_version} to redis failed.")
else:
LOGGER.info(
f"Store model {model_id} {model_version} to redis successfully."
f"Archive path: {model.archive_model_file_path} Key: {store_key}"
)
def get_predict_conf():
request_data = request.json
required_parameters = ['model_id', 'model_version']
check_config(request_data, required_parameters)
model_dir = os.path.join(get_project_base_directory(), 'model_local_cache')
model_fp_list = glob.glob(
model_dir +
f"/guest#*#{request_data['model_id']}/{request_data['model_version']}")
if model_fp_list:
fp = model_fp_list[0]
pipeline_model = PipelinedModel(model_id=fp.split('/')[-2],
model_version=fp.split('/')[-1])
pipeline = pipeline_model.read_component_model('pipeline',
'pipeline')['Pipeline']
predict_dsl = json_loads(pipeline.inference_dsl)
train_runtime_conf = json_loads(pipeline.train_runtime_conf)
parser = schedule_utils.get_dsl_parser_by_version(
train_runtime_conf.get('dsl_version', '1'))
predict_conf = parser.generate_predict_conf_template(
predict_dsl=predict_dsl,
train_conf=train_runtime_conf,
model_id=request_data['model_id'],
model_version=request_data['model_version'])
else:
predict_conf = ''
if predict_conf:
if request_data.get("filename"):
os.makedirs(TEMP_DIRECTORY, exist_ok=True)
temp_filepath = os.path.join(TEMP_DIRECTORY,
request_data.get("filename"))
with open(temp_filepath, "w") as fout:
fout.write(json_dumps(predict_conf, indent=4))
return send_file(open(temp_filepath, "rb"),
as_attachment=True,
attachment_filename=request_data.get("filename"))
else:
return get_json_result(data=predict_conf)
return error_response(
210,
"No model found, please check if arguments are specified correctly.")
def store(self, model_id: str, model_version: str, store_address: dict, force_update: bool = False):
"""
Store the model from local cache to mysql
:param model_id:
:param model_version:
:param store_address:
:param force_update:
:return:
"""
try:
self.get_connection(config=store_address)
DB.create_tables([MachineLearningModel])
model = PipelinedModel(model_id=model_id, model_version=model_version)
LOGGER.info("start store model {} {}".format(model_id, model_version))
with DB.connection_context():
with open(model.packaging_model(), "rb") as fr:
slice_index = 0
while True:
content = fr.read(SLICE_MAX_SIZE)
if content:
model_in_table = MachineLearningModel()
model_in_table.f_create_time = current_timestamp()
model_in_table.f_model_id = model_id
model_in_table.f_model_version = model_version
model_in_table.f_content = serialize_b64(content, to_str=True)
model_in_table.f_size = sys.getsizeof(model_in_table.f_content)
model_in_table.f_slice_index = slice_index
if force_update:
model_in_table.save(only=[MachineLearningModel.f_content, MachineLearningModel.f_size,
MachineLearningModel.f_update_time, MachineLearningModel.f_slice_index])
LOGGER.info("update model {} {} slice index {} content".format(model_id, model_version, slice_index))
else:
model_in_table.save(force_insert=True)
slice_index += 1
LOGGER.info("insert model {} {} slice index {} content".format(model_id, model_version, slice_index))
else:
break
LOGGER.info("Store model {} {} to mysql successfully".format(model_id, model_version))
self.close_connection()
except Exception as e:
LOGGER.exception(e)
raise Exception("Store model {} {} to mysql failed".format(model_id, model_version))
def read_component_model(self):
pipelined_model = PipelinedModel(
gen_party_model_id(self.model_id, self.tracker.role,
self.tracker.party_id), self.model_version)
component_model = pipelined_model._read_component_model(
self.component_name, self.model_alias)
if not component_model:
raise ValueError('The component model is empty.')
self.model_output = component_model
self.tracker.set_metric_meta(
'model_loader', f'{self.component_name}-{self.model_alias}',
MetricMeta(
'component_model', 'component_model_info', {
'model_id': self.model_id,
'model_version': self.model_version,
'component_name': self.component_name,
'model_alias': self.model_alias,
}))
def gather_model_info_data(model: PipelinedModel, query_filters=None):
if model.exists():
pipeline = model.read_pipeline_model()
model_info = OrderedDict()
if query_filters and isinstance(query_filters, list):
for attr, field in pipeline.ListFields():
if attr.name in query_filters:
if isinstance(field, bytes):
model_info["f_" + attr.name] = json_loads(
field, OrderedDict)
else:
model_info["f_" + attr.name] = field
else:
for attr, field in pipeline.ListFields():
if isinstance(field, bytes):
model_info["f_" + attr.name] = json_loads(
field, OrderedDict)
else:
model_info["f_" + attr.name] = field
return model_info
return []
def store(self,
model_id: str,
model_version: str,
store_address: dict,
force_update: bool = False):
"""
Store the model from local cache to cos
:param model_id:
:param model_version:
:param store_address:
:param force_update:
:return:
"""
store_key = self.store_key(model_id, model_version) + '.zip'
if not force_update and self.exists(model_id, model_version,
store_address):
raise FileExistsError(f"The object {store_key} already exists.")
model = PipelinedModel(model_id, model_version)
cos = self.get_connection(store_address)
try:
model.packaging_model()
response = cos.upload_file(
Bucket=store_address["Bucket"],
LocalFilePath=model.archive_model_file_path,
Key=store_key,
EnableMD5=True,
)
except Exception as e:
LOGGER.exception(e)
raise Exception(
f"Store model {model_id} {model_version} to Tencent COS failed."
)
else:
LOGGER.info(
f"Store model {model_id} {model_version} to Tencent COS successfully. "
f"Archive path: {model.archive_model_file_path} Key: {store_key} ETag: {response['ETag']}"
)
def output_model_reload(cls, job, source_job):
source_model_id = model_utils.gen_party_model_id(
source_job.f_runtime_conf.get("job_parameters").get("common").get(
"model_id"), job.f_role, job.f_party_id)
model_id = model_utils.gen_party_model_id(
job.f_runtime_conf.get("job_parameters").get("common").get(
"model_id"), job.f_role, job.f_party_id)
PipelinedModel(
model_id=model_id,
model_version=job.f_job_id).reload_component_model(
model_id=source_model_id,
model_version=job.f_inheritance_info.get("job_id"),
component_list=job.f_inheritance_info.get("component_list"))
def restore(self, model_id: str, model_version: str, store_address: dict):
"""
Restore model from mysql to local cache
:param model_id:
:param model_version:
:param store_address:
:return:
"""
model = PipelinedModel(model_id, model_version)
self.get_connection(store_address)
try:
with DB.connection_context():
models_in_tables = MachineLearningModel.select().where(
MachineLearningModel.f_model_id == model_id,
MachineLearningModel.f_model_version == model_version,
).order_by(MachineLearningModel.f_slice_index)
if not models_in_tables:
raise ValueError(f"Cannot found model in table.")
model_archive_data = b''.join(
deserialize_b64(models_in_table.f_content)
for models_in_table in models_in_tables)
if not model_archive_data:
raise ValueError(f"Cannot get model archive data.")
with open(model.archive_model_file_path, "wb") as fw:
fw.write(model_archive_data)
model.unpack_model(model.archive_model_file_path)
except Exception as e:
LOGGER.exception(e)
raise Exception(
f"Restore model {model_id} {model_version} from mysql failed.")
else:
LOGGER.info(
f"Restore model to {model.archive_model_file_path} from mysql successfully."
)
finally:
self.close_connection()
def restore(self, model_id: str, model_version: str, store_address: dict):
"""
Restore model from mysql to local cache
:param model_id:
:param model_version:
:param store_address:
:return:
"""
try:
self.get_connection(config=store_address)
model = PipelinedModel(model_id=model_id, model_version=model_version)
with DB.connection_context():
models_in_tables = MachineLearningModel.select().where(MachineLearningModel.f_model_id == model_id,
MachineLearningModel.f_model_version == model_version).\
order_by(MachineLearningModel.f_slice_index)
if not models_in_tables:
raise Exception("Restore model {} {} from mysql failed: {}".format(
model_id, model_version, "can not found model in table"))
f_content = ''
for models_in_table in models_in_tables:
if not f_content:
f_content = models_in_table.f_content
else:
f_content += models_in_table.f_content
model_archive_data = deserialize_b64(f_content)
if not model_archive_data:
raise Exception("Restore model {} {} from mysql failed: {}".format(
model_id, model_version, "can not get model archive data"))
with open(model.archive_model_file_path, "wb") as fw:
fw.write(model_archive_data)
model.unpack_model(model.archive_model_file_path)
LOGGER.info("Restore model to {} from mysql successfully".format(model.archive_model_file_path))
self.close_connection()
except Exception as e:
LOGGER.exception(e)
raise Exception("Restore model {} {} from mysql failed".format(model_id, model_version))
def store(self,
model_id: str,
model_version: str,
store_address: dict,
force_update: bool = False):
"""
Store the model from local cache to mysql
:param model_id:
:param model_version:
:param store_address:
:param force_update:
:return:
"""
if not force_update and self.exists(model_id, model_version,
store_address):
raise FileExistsError(
f"The model {model_id} {model_version} already exists in the database."
)
model = PipelinedModel(model_id, model_version)
self.get_connection(store_address)
try:
DB.create_tables([MachineLearningModel])
LOGGER.info(f"Starting store model {model_id} {model_version}.")
with open(model.packaging_model(),
"rb") as fr, DB.connection_context():
slice_index = 0
while True:
content = fr.read(SLICE_MAX_SIZE)
if not content:
break
model_in_table = MachineLearningModel()
model_in_table.f_create_time = current_timestamp()
model_in_table.f_model_id = model_id
model_in_table.f_model_version = model_version
model_in_table.f_content = serialize_b64(content,
to_str=True)
model_in_table.f_size = sys.getsizeof(
model_in_table.f_content)
model_in_table.f_slice_index = slice_index
rows = 0
if force_update:
rows = model_in_table.save(only=[
MachineLearningModel.f_content,
MachineLearningModel.f_size,
MachineLearningModel.f_update_time,
MachineLearningModel.f_slice_index,
])
if not rows:
rows = model_in_table.save(force_insert=True)
if not rows:
raise Exception(
f"Save slice index {slice_index} failed")
LOGGER.info(
f"Saved slice index {slice_index} of model {model_id} {model_version}."
)
slice_index += 1
except Exception as e:
LOGGER.exception(e)
raise Exception(
f"Store model {model_id} {model_version} to mysql failed.")
else:
LOGGER.info(
f"Store model {model_id} {model_version} to mysql successfully."
)
finally:
self.close_connection()
def deploy(config_data):
model_id = config_data.get('model_id')
model_version = config_data.get('model_version')
local_role = config_data.get('local').get('role')
local_party_id = config_data.get('local').get('party_id')
child_model_version = config_data.get('child_model_version')
components_checkpoint = config_data.get('components_checkpoint', {})
warning_msg = ""
try:
party_model_id = gen_party_model_id(model_id=model_id,
role=local_role,
party_id=local_party_id)
model = PipelinedModel(model_id=party_model_id,
model_version=model_version)
model_data = model.collect_models(in_bytes=True)
if "pipeline.pipeline:Pipeline" not in model_data:
raise Exception("Can not found pipeline file in model.")
# check if the model could be executed the deploy process (parent/child)
if not check_before_deploy(model):
raise Exception('Child model could not be deployed.')
# copy proto content from parent model and generate a child model
deploy_model = PipelinedModel(model_id=party_model_id,
model_version=child_model_version)
shutil.copytree(src=model.model_path,
dst=deploy_model.model_path,
ignore=lambda src, names: {'checkpoint'}
if src == model.model_path else {})
pipeline_model = deploy_model.read_pipeline_model()
train_runtime_conf = json_loads(pipeline_model.train_runtime_conf)
runtime_conf_on_party = json_loads(
pipeline_model.runtime_conf_on_party)
dsl_version = train_runtime_conf.get("dsl_version", "1")
parser = get_dsl_parser_by_version(dsl_version)
train_dsl = json_loads(pipeline_model.train_dsl)
parent_predict_dsl = json_loads(pipeline_model.inference_dsl)
if config_data.get('dsl') or config_data.get('predict_dsl'):
inference_dsl = config_data.get('dsl') if config_data.get(
'dsl') else config_data.get('predict_dsl')
if not isinstance(inference_dsl, dict):
inference_dsl = json_loads(inference_dsl)
else:
if config_data.get('cpn_list', None):
cpn_list = config_data.pop('cpn_list')
else:
cpn_list = list(train_dsl.get('components', {}).keys())
if int(dsl_version) == 1:
# convert v1 dsl to v2 dsl
inference_dsl, warning_msg = parser.convert_dsl_v1_to_v2(
parent_predict_dsl)
else:
parser = get_dsl_parser_by_version(dsl_version)
inference_dsl = parser.deploy_component(cpn_list, train_dsl)
# convert v1 conf to v2 conf
if int(dsl_version) == 1:
components = parser.get_components_light_weight(inference_dsl)
from fate_flow.db.component_registry import ComponentRegistry
job_providers = parser.get_job_providers(
dsl=inference_dsl, provider_detail=ComponentRegistry.REGISTRY)
cpn_role_parameters = dict()
for cpn in components:
cpn_name = cpn.get_name()
role_params = parser.parse_component_role_parameters(
component=cpn_name,
dsl=inference_dsl,
runtime_conf=train_runtime_conf,
provider_detail=ComponentRegistry.REGISTRY,
provider_name=job_providers[cpn_name]["provider"]["name"],
provider_version=job_providers[cpn_name]["provider"]
["version"])
cpn_role_parameters[cpn_name] = role_params
train_runtime_conf = parser.convert_conf_v1_to_v2(
train_runtime_conf, cpn_role_parameters)
adapter = JobRuntimeConfigAdapter(train_runtime_conf)
train_runtime_conf = adapter.update_model_id_version(
model_version=deploy_model.model_version)
pipeline_model.model_version = child_model_version
pipeline_model.train_runtime_conf = json_dumps(train_runtime_conf,
byte=True)
# save inference dsl into child model file
parser = get_dsl_parser_by_version(2)
parser.verify_dsl(inference_dsl, "predict")
inference_dsl = JobSaver.fill_job_inference_dsl(
job_id=model_version,
role=local_role,
party_id=local_party_id,
dsl_parser=parser,
origin_inference_dsl=inference_dsl)
pipeline_model.inference_dsl = json_dumps(inference_dsl, byte=True)
if compare_version(pipeline_model.fate_version, '1.5.0') == 'gt':
pipeline_model.parent_info = json_dumps(
{
'parent_model_id': model_id,
'parent_model_version': model_version
},
byte=True)
pipeline_model.parent = False
runtime_conf_on_party['job_parameters'][
'model_version'] = child_model_version
pipeline_model.runtime_conf_on_party = json_dumps(
runtime_conf_on_party, byte=True)
# save model file
deploy_model.save_pipeline(pipeline_model)
shutil.copyfile(
os.path.join(deploy_model.model_path, "pipeline.pb"),
os.path.join(deploy_model.model_path, "variables", "data",
"pipeline", "pipeline", "Pipeline"))
model_info = gather_model_info_data(deploy_model)
model_info['job_id'] = model_info['f_model_version']
model_info['size'] = deploy_model.calculate_model_file_size()
model_info['role'] = local_role
model_info['party_id'] = local_party_id
model_info['parent'] = False if model_info.get(
'f_inference_dsl') else True
if compare_version(model_info['f_fate_version'], '1.5.0') == 'eq':
model_info['roles'] = model_info.get('f_train_runtime_conf',
{}).get('role', {})
model_info['initiator_role'] = model_info.get(
'f_train_runtime_conf', {}).get('initiator', {}).get('role')
model_info['initiator_party_id'] = model_info.get(
'f_train_runtime_conf', {}).get('initiator',
{}).get('party_id')
save_model_info(model_info)
for component_name, component in train_dsl.get('components',
{}).items():
step_index = components_checkpoint.get(component_name,
{}).get('step_index')
step_name = components_checkpoint.get(component_name,
{}).get('step_name')
if step_index is not None:
step_index = int(step_index)
step_name = None
elif step_name is None:
continue
checkpoint_manager = CheckpointManager(
role=local_role,
party_id=local_party_id,
model_id=model_id,
model_version=model_version,
component_name=component_name,
mkdir=False,
)
checkpoint_manager.load_checkpoints_from_disk()
if checkpoint_manager.latest_checkpoint is not None:
checkpoint_manager.deploy(
child_model_version,
component['output']['model'][0] if component.get(
'output', {}).get('model') else 'default',
step_index,
step_name,
)
except Exception as e:
stat_logger.exception(e)
return 100, f"deploy model of role {local_role} {local_party_id} failed, details: {str(e)}"
else:
msg = f"deploy model of role {local_role} {local_party_id} success"
if warning_msg:
msg = msg + f", warning: {warning_msg}"
return 0, msg
class TestPipelinedModel(unittest.TestCase):
def setUp(self):
shutil.rmtree(TEMP_DIRECTORY, True)
self.pipelined_model = PipelinedModel('foobar', 'v1')
shutil.rmtree(self.pipelined_model.model_path, True)
self.pipelined_model.create_pipelined_model()
with open(self.pipelined_model.define_meta_path, 'w',
encoding='utf8') as f:
yaml.dump(data_define_meta, f)
def tearDown(self):
shutil.rmtree(TEMP_DIRECTORY, True)
shutil.rmtree(self.pipelined_model.model_path, True)
def test_write_read_file_same_time(self):
fw = open(self.pipelined_model.define_meta_path, 'r+', encoding='utf8')
self.assertEqual(yaml.safe_load(fw), data_define_meta)
fw.seek(0)
fw.write('foobar')
with open(self.pipelined_model.define_meta_path,
encoding='utf8') as fr:
self.assertEqual(yaml.safe_load(fr), data_define_meta)
fw.truncate()
with open(self.pipelined_model.define_meta_path,
encoding='utf8') as fr:
self.assertEqual(fr.read(), 'foobar')
fw.seek(0)
fw.write('abc')
fw.close()
with open(self.pipelined_model.define_meta_path,
encoding='utf8') as fr:
self.assertEqual(fr.read(), 'abcbar')
def test_update_component_meta_with_changes(self):
with patch('ruamel.yaml.dump', side_effect=yaml.dump) as yaml_dump:
self.pipelined_model.update_component_meta(
'dataio_0', 'DataIO_v0', 'dataio', {
'DataIOMeta': 'DataIOMeta_v0',
'DataIOParam': 'DataIOParam_v0',
})
yaml_dump.assert_called_once()
with open(self.pipelined_model.define_meta_path,
encoding='utf8') as tmp:
define_index = yaml.safe_load(tmp)
_data = deepcopy(data_define_meta)
_data['component_define']['dataio_0']['module_name'] = 'DataIO_v0'
_data['model_proto']['dataio_0']['dataio'] = {
'DataIOMeta': 'DataIOMeta_v0',
'DataIOParam': 'DataIOParam_v0',
}
self.assertEqual(define_index, _data)
def test_update_component_meta_without_changes(self):
with open(self.pipelined_model.define_meta_path, 'w',
encoding='utf8') as f:
yaml.dump(data_define_meta, f, Dumper=yaml.RoundTripDumper)
with patch('ruamel.yaml.dump', side_effect=yaml.dump) as yaml_dump:
self.pipelined_model.update_component_meta(
*args_update_component_meta)
yaml_dump.assert_not_called()
with open(self.pipelined_model.define_meta_path,
encoding='utf8') as tmp:
define_index = yaml.safe_load(tmp)
self.assertEqual(define_index, data_define_meta)
def test_update_component_meta_multi_thread(self):
with patch('ruamel.yaml.safe_load', side_effect=yaml.safe_load) as yaml_load, \
patch('ruamel.yaml.dump', side_effect=yaml.dump) as yaml_dump, \
concurrent.futures.ThreadPoolExecutor(max_workers=100) as executor:
for _ in range(100):
executor.submit(self.pipelined_model.update_component_meta,
*args_update_component_meta)
self.assertEqual(yaml_load.call_count, 100)
self.assertEqual(yaml_dump.call_count, 0)
with open(self.pipelined_model.define_meta_path,
encoding='utf8') as tmp:
define_index = yaml.safe_load(tmp)
self.assertEqual(define_index, data_define_meta)
def test_update_component_meta_empty_file(self):
open(self.pipelined_model.define_meta_path, 'w').close()
with self.assertRaisesRegex(ValueError, 'Invalid meta file'):
self.pipelined_model.update_component_meta(
*args_update_component_meta)
def test_packaging_model(self):
archive_file_path = self.pipelined_model.packaging_model()
self.assertEqual(archive_file_path,
self.pipelined_model.archive_model_file_path)
self.assertTrue(Path(archive_file_path).is_file())
self.assertTrue(Path(archive_file_path + '.sha1').is_file())
with ZipFile(archive_file_path) as z:
with io.TextIOWrapper(z.open('define/define_meta.yaml'),
encoding='utf8') as f:
define_index = yaml.safe_load(f)
self.assertEqual(define_index, data_define_meta)
with open(archive_file_path,
'rb') as f, open(archive_file_path + '.sha1',
encoding='utf8') as g:
sha1 = hashlib.sha1(f.read()).hexdigest()
sha1_orig = g.read().strip()
self.assertEqual(sha1, sha1_orig)
def test_packaging_model_not_exists(self):
shutil.rmtree(self.pipelined_model.model_path, True)
with self.assertRaisesRegex(
FileNotFoundError,
'Can not found foobar v1 model local cache'):
self.pipelined_model.packaging_model()
def test_unpack_model(self):
archive_file_path = self.pipelined_model.packaging_model()
self.assertTrue(Path(archive_file_path + '.sha1').is_file())
shutil.rmtree(self.pipelined_model.model_path, True)
self.assertFalse(Path(self.pipelined_model.model_path).exists())
self.pipelined_model.unpack_model(archive_file_path)
with open(self.pipelined_model.define_meta_path,
encoding='utf8') as tmp:
define_index = yaml.safe_load(tmp)
self.assertEqual(define_index, data_define_meta)
def test_unpack_model_local_cache_exists(self):
archive_file_path = self.pipelined_model.packaging_model()
with self.assertRaisesRegex(
FileExistsError,
'Model foobar v1 local cache already existed'):
self.pipelined_model.unpack_model(archive_file_path)
def test_unpack_model_no_hash_file(self):
archive_file_path = self.pipelined_model.packaging_model()
Path(archive_file_path + '.sha1').unlink()
self.assertFalse(Path(archive_file_path + '.sha1').exists())
shutil.rmtree(self.pipelined_model.model_path, True)
self.assertFalse(os.path.exists(self.pipelined_model.model_path))
self.pipelined_model.unpack_model(archive_file_path)
with open(self.pipelined_model.define_meta_path,
encoding='utf8') as tmp:
define_index = yaml.safe_load(tmp)
self.assertEqual(define_index, data_define_meta)
def test_unpack_model_hash_not_match(self):
archive_file_path = self.pipelined_model.packaging_model()
self.assertTrue(Path(archive_file_path + '.sha1').is_file())
with open(archive_file_path + '.sha1', 'w', encoding='utf8') as f:
f.write('abc123')
shutil.rmtree(self.pipelined_model.model_path, True)
self.assertFalse(Path(self.pipelined_model.model_path).exists())
with self.assertRaisesRegex(ValueError, 'Hash not match.'):
self.pipelined_model.unpack_model(archive_file_path)
def deploy(config_data):
model_id = config_data.get('model_id')
model_version = config_data.get('model_version')
local_role = config_data.get('local').get('role')
local_party_id = config_data.get('local').get('party_id')
child_model_version = config_data.get('child_model_version')
try:
party_model_id = model_utils.gen_party_model_id(
model_id=model_id, role=local_role, party_id=local_party_id)
model = PipelinedModel(model_id=party_model_id,
model_version=model_version)
model_data = model.collect_models(in_bytes=True)
if "pipeline.pipeline:Pipeline" not in model_data:
raise Exception("Can not found pipeline file in model.")
# check if the model could be executed the deploy process (parent/child)
if not check_before_deploy(model):
raise Exception('Child model could not be deployed.')
# copy proto content from parent model and generate a child model
deploy_model = PipelinedModel(model_id=party_model_id,
model_version=child_model_version)
shutil.copytree(src=model.model_path, dst=deploy_model.model_path)
pipeline = deploy_model.read_component_model('pipeline',
'pipeline')['Pipeline']
# modify two pipeline files (model version/ train_runtime_conf)
train_runtime_conf = json_loads(pipeline.train_runtime_conf)
adapter = JobRuntimeConfigAdapter(train_runtime_conf)
train_runtime_conf = adapter.update_model_id_version(
model_version=deploy_model.model_version)
pipeline.model_version = child_model_version
pipeline.train_runtime_conf = json_dumps(train_runtime_conf, byte=True)
parser = get_dsl_parser_by_version(
train_runtime_conf.get('dsl_version', '1'))
train_dsl = json_loads(pipeline.train_dsl)
parent_predict_dsl = json_loads(pipeline.inference_dsl)
if str(train_runtime_conf.get('dsl_version', '1')) == '1':
predict_dsl = json_loads(pipeline.inference_dsl)
else:
if config_data.get('dsl') or config_data.get('predict_dsl'):
predict_dsl = config_data.get('dsl') if config_data.get(
'dsl') else config_data.get('predict_dsl')
if not isinstance(predict_dsl, dict):
predict_dsl = json_loads(predict_dsl)
else:
if config_data.get('cpn_list', None):
cpn_list = config_data.pop('cpn_list')
else:
cpn_list = list(train_dsl.get('components', {}).keys())
parser_version = train_runtime_conf.get('dsl_version', '1')
if str(parser_version) == '1':
predict_dsl = parent_predict_dsl
else:
parser = schedule_utils.get_dsl_parser_by_version(
parser_version)
predict_dsl = parser.deploy_component(cpn_list, train_dsl)
# save predict dsl into child model file
parser.verify_dsl(predict_dsl, "predict")
inference_dsl = parser.get_predict_dsl(
role=local_role,
predict_dsl=predict_dsl,
setting_conf_prefix=file_utils.
get_federatedml_setting_conf_directory())
pipeline.inference_dsl = json_dumps(inference_dsl, byte=True)
if model_utils.compare_version(pipeline.fate_version, '1.5.0') == 'gt':
pipeline.parent_info = json_dumps(
{
'parent_model_id': model_id,
'parent_model_version': model_version
},
byte=True)
pipeline.parent = False
runtime_conf_on_party = json_loads(pipeline.runtime_conf_on_party)
runtime_conf_on_party['job_parameters'][
'model_version'] = child_model_version
pipeline.runtime_conf_on_party = json_dumps(runtime_conf_on_party,
byte=True)
# save model file
deploy_model.save_pipeline(pipeline)
shutil.copyfile(
os.path.join(deploy_model.model_path, "pipeline.pb"),
os.path.join(deploy_model.model_path, "variables", "data",
"pipeline", "pipeline", "Pipeline"))
model_info = model_utils.gather_model_info_data(deploy_model)
model_info['job_id'] = model_info['f_model_version']
model_info['size'] = deploy_model.calculate_model_file_size()
model_info['role'] = local_role
model_info['party_id'] = local_party_id
model_info['work_mode'] = adapter.get_job_work_mode()
model_info['parent'] = False if model_info.get(
'f_inference_dsl') else True
if model_utils.compare_version(model_info['f_fate_version'],
'1.5.0') == 'eq':
model_info['roles'] = model_info.get('f_train_runtime_conf',
{}).get('role', {})
model_info['initiator_role'] = model_info.get(
'f_train_runtime_conf', {}).get('initiator', {}).get('role')
model_info['initiator_party_id'] = model_info.get(
'f_train_runtime_conf', {}).get('initiator',
{}).get('party_id')
model_utils.save_model_info(model_info)
except Exception as e:
stat_logger.exception(e)
return 100, f"deploy model of role {local_role} {local_party_id} failed, details: {str(e)}"
else:
return 0, f"deploy model of role {local_role} {local_party_id} success"