def load_data_io_model(model_name="DataIO",
model_table=None,
model_namespace=None):
model_meta = DataIOMeta()
model_param = DataIOParam()
manager.read_model(buffer_type=model_name + ".meta",
proto_buffer=model_meta,
name=model_table,
namespace=model_namespace)
manager.read_model(buffer_type=model_name + ".param",
proto_buffer=model_param,
name=model_table,
namespace=model_namespace)
delimitor = model_meta.delimitor
data_type = model_meta.data_type
with_label = model_meta.with_label
label_idx = model_meta.label_idx
label_type = model_meta.label_type
output_format = model_meta.output_format
header = list(model_param.header)
return delimitor, data_type, with_label, label_idx, label_type, output_format, header
def load_model(self, name, namespace):
"""
Load OneVsRest model
"""
model_obj = one_vs_rest_param_pb2.OneVsRestParam()
buffer_type = "{}.param".format(self.class_name)
model_manager.read_model(buffer_type=buffer_type,
proto_buffer=model_obj,
name=name,
namespace=namespace)
LOGGER.info("OneVsRest classes number:{}".format(len(
model_obj.classes)))
self.models = []
for i, classifier_model in enumerate(model_obj.classifier_models):
classifier = copy.deepcopy(self.classifier)
classifier.load_model(classifier_model.name,
classifier_model.namespace)
self.models.append(classifier)
LOGGER.info("finish load model_{}, classes is {}".format(
i, model_obj.classes[i]))
self.classes = []
for model_class in model_obj.classes:
self.classes.append(model_class)
LOGGER.info("finish load OneVsRest model.")
def load_model(self, name, namespace, header):
"""
load Scaler model
Parameters
----------
name: str, DTable table_name
namespace: str, DTable namespace
Returns
----------
list,
scale information
"""
self.header = header
param_buffer_type = "{}.param".format(self.class_name)
param_obj = feature_scale_param_pb2.ScaleParam()
model_manager.read_model(buffer_type=param_buffer_type,
proto_buffer=param_obj,
name=name,
namespace=namespace)
if self.scale_param.method == consts.MINMAXSCALE:
cols_scale_value = []
param_dict = param_obj.minmax_scale_param
for idx, header_name in enumerate(self.header):
if header_name in param_dict:
feat_upper = param_dict[header_name].feat_upper
feat_lower = param_dict[header_name].feat_lower
out_upper = param_dict[header_name].out_upper
out_lower = param_dict[header_name].out_lower
cols_scale_value.append((feat_lower,
feat_upper,
out_lower,
out_upper))
else:
raise ValueError("Can not find the header name {} in model.".format(header_name))
model_scale_value_results = [ cols_scale_value ]
elif self.scale_param.method == consts.STANDARDSCALE:
mean = []
std = []
param_dict = param_obj.standard_scale_param
for idx, header_name in enumerate(self.header):
if header_name in param_dict:
mean.append(param_dict[header_name].mean)
std.append(param_dict[header_name].scale)
else:
raise ValueError("Can not find the header name {} in model.".format(header_name))
model_scale_value_results = [ mean, std ]
else:
raise ValueError("Unknown scale method:{}".format(self.scale_param.method))
return model_scale_value_results
def load_model(self, name, namespace):
result_obj = feature_selection_param_pb2.FeatureSelectionParam()
model_manager.read_model(
buffer_type="HeteroFeatureSelection{}.param".format(
self.party_name),
proto_buffer=result_obj,
name=name,
namespace=namespace)
self.results = list(result_obj.results)
left_col_obj = result_obj.final_left_cols
self.cols = list(left_col_obj.original_cols)
self.left_cols = dict(left_col_obj.left_cols)
def load_model(self, name, namespace):
result_obj = feature_selection_param_pb2.FeatureSelectionParam()
model_manager.read_model(
buffer_type="HeteroFeatureSelectionGuest.param",
proto_buffer=result_obj,
name=name,
namespace=namespace)
self.results = list(result_obj.results)
if len(self.results) == 0:
self.left_cols = -1
else:
result_obj = self.results[-1]
self.left_cols = list(result_obj.left_cols)
def load_model(self, model_table, model_namespace):
LOGGER.info("load model")
model_meta = BoostingTreeModelMeta()
manager.read_model(buffer_type="HeteroSecureBoostingTreeGuest.meta",
proto_buffer=model_meta,
name=model_table,
namespace=model_namespace)
self.set_model_meta(model_meta)
model_param = BoostingTreeModelParam()
manager.read_model(buffer_type="HeteroSecureBoostingTreeGuest.param",
proto_buffer=model_param,
name=model_table,
namespace=model_namespace)
self.set_model_param(model_param)
def load_model(self, name, namespace):
result_obj = feature_binning_param_pb2.FeatureBinningParam()
return_code = model_manager.read_model(buffer_type="HeteroFeatureBinning{}.param".format(self.party_name),
proto_buffer=result_obj,
name=name,
namespace=namespace)
binning_result_obj = dict(result_obj.binning_result.binning_result)
host_params = dict(result_obj.host_results)
# LOGGER.debug("Party name is :{}".format(self.party_name))
# LOGGER.debug('Loading model, binning_result_obj is : {}'.format(binning_result_obj))
self.binning_result = {}
self.host_results = {}
self.cols = []
for col_name, iv_attr_obj in binning_result_obj.items():
iv_attr = IVAttributes([], [], [], [], [], [])
iv_attr.reconstruct(iv_attr_obj)
self.binning_result[col_name] = iv_attr
self.cols.append(col_name)
for host_name, host_result_obj in host_params.items():
host_result_obj = dict(host_result_obj.binning_result)
for col_name, iv_attr_obj in host_result_obj.items():
iv_attr = IVAttributes([], [], [], [], [], [])
iv_attr.reconstruct(iv_attr_obj)
host_result_obj[col_name] = iv_attr
self.host_results[host_name] = host_result_obj
return return_code
def load_model(self, name, namespace):
result_obj = lr_model_param_pb2.LRModelParam()
buffer_type = "{}.param".format(self.class_name)
model_manager.read_model(buffer_type=buffer_type,
proto_buffer=result_obj,
name=name,
namespace=namespace)
self.header = list(result_obj.header)
feature_shape = len(self.header)
self.coef_ = np.zeros(feature_shape)
weight_dict = dict(result_obj.weight)
self.intercept_ = result_obj.intercept
for idx, header_name in enumerate(self.header):
self.coef_[idx] = weight_dict.get(header_name)
def _load_pipeline(self):
buffer_type = "Pipeline"
pipeline_obj = pipeline_pb2.Pipeline()
pipeline_obj = model_manager.read_model(buffer_type=buffer_type,
proto_buffer=pipeline_obj,
name=self.workflow_param.model_table,
namespace=self.workflow_param.model_namespace)
pipeline_obj.node_meta = list(pipeline_obj.node_meta)
pipeline_obj.node_param = list(pipeline_obj.node_param)
self.pipeline = pipeline_obj
def load_outlier_model(header=None,
model_name="Outlier",
model_table=None,
model_namespace=None):
model_meta = OutlierMeta()
model_param = OutlierParam()
manager.read_model(buffer_type=model_name + ".meta",
proto_buffer=model_meta,
name=model_table,
namespace=model_namespace)
manager.read_model(buffer_type=model_name + ".param",
proto_buffer=model_param,
name=model_table,
namespace=model_namespace)
outlier_replace = model_meta.is_outlier
outlier_replace_method = model_meta.strategy
outlier_value = model_meta.outlier_value
outlier_replace_value = model_param.outlier_replace_value
if outlier_replace:
if not outlier_replace_method:
outlier_replace_method = None
if not outlier_value:
outlier_value = None
else:
outlier_value = list(outlier_value)
if outlier_replace_value:
outlier_replace_value = [
outlier_replace_value.get(head) for head in header
]
else:
outlier_replace_value = None
else:
outlier_replace_method = None
outlier_value = None
outlier_replace_value = None
return outlier_replace, outlier_replace_method, outlier_value, outlier_replace_value
def load_missing_imputer_model(header=None,
model_name="Imputer",
model_table=None,
model_namespace=None):
model_meta = ImputerMeta()
model_param = ImputerParam()
manager.read_model(buffer_type=model_name + ".meta",
proto_buffer=model_meta,
name=model_table,
namespace=model_namespace)
manager.read_model(buffer_type=model_name + ".param",
proto_buffer=model_param,
name=model_table,
namespace=model_namespace)
missing_fill = model_meta.is_imputer
missing_replace_method = model_meta.strategy
missing_value = model_meta.missing_value
missing_fill_value = model_param.missing_replace_value
if missing_fill:
if not missing_replace_method:
missing_replace_method = None
if not missing_value:
missing_value = None
else:
missing_value = list(missing_value)
if missing_fill_value:
missing_fill_value = [
missing_fill_value.get(head) for head in header
]
else:
missing_fill_value = None
else:
missing_replace_method = None
missing_value = None
missing_fill_value = None
return missing_fill, missing_replace_method, missing_value, missing_fill_value
def load_model(self, name, namespace):
result_obj = onehot_param_pb2.OneHotParam()
return_code = model_manager.read_model(buffer_type='OneHotEncoder.param',
proto_buffer=result_obj,
name=name,
namespace=namespace)
self.col_maps = dict(result_obj.col_map)
for k, v in self.col_maps.items():
self.col_maps[k] = dict(v.encode_map)
return return_code
def load_model(self, name, namespace):
result_obj = feature_binning_param_pb2.FeatureBinningParam()
model_manager.read_model(buffer_type="HeteroFeatureBinningGuest.param",
proto_buffer=result_obj,
name=name,
namespace=namespace)
self.iv_attrs = []
for iv_dict in list(result_obj.iv_result):
iv_attr = IVAttributes([], [], [], [], [], [], [])
iv_attr.reconstruct(iv_dict)
self.iv_attrs.append(iv_attr)
self.host_iv_attrs = []
for iv_dict in list(result_obj.host_iv_result):
iv_attr = IVAttributes([], [], [], [], [], [], [])
iv_attr.reconstruct(iv_dict)
self.host_iv_attrs.append(iv_attr)
self.cols = list(result_obj.cols)
