def load_data_io_model(model_name="DataIO", model_table=None, model_namespace=None): model_meta = DataIOMeta() model_param = DataIOParam() manager.read_model(buffer_type=model_name + ".meta", proto_buffer=model_meta, name=model_table, namespace=model_namespace) manager.read_model(buffer_type=model_name + ".param", proto_buffer=model_param, name=model_table, namespace=model_namespace) delimitor = model_meta.delimitor data_type = model_meta.data_type with_label = model_meta.with_label label_idx = model_meta.label_idx label_type = model_meta.label_type output_format = model_meta.output_format header = list(model_param.header) return delimitor, data_type, with_label, label_idx, label_type, output_format, header
def load_model(self, name, namespace): """ Load OneVsRest model """ model_obj = one_vs_rest_param_pb2.OneVsRestParam() buffer_type = "{}.param".format(self.class_name) model_manager.read_model(buffer_type=buffer_type, proto_buffer=model_obj, name=name, namespace=namespace) LOGGER.info("OneVsRest classes number:{}".format(len( model_obj.classes))) self.models = [] for i, classifier_model in enumerate(model_obj.classifier_models): classifier = copy.deepcopy(self.classifier) classifier.load_model(classifier_model.name, classifier_model.namespace) self.models.append(classifier) LOGGER.info("finish load model_{}, classes is {}".format( i, model_obj.classes[i])) self.classes = [] for model_class in model_obj.classes: self.classes.append(model_class) LOGGER.info("finish load OneVsRest model.")
def load_model(self, name, namespace, header): """ load Scaler model Parameters ---------- name: str, DTable table_name namespace: str, DTable namespace Returns ---------- list, scale information """ self.header = header param_buffer_type = "{}.param".format(self.class_name) param_obj = feature_scale_param_pb2.ScaleParam() model_manager.read_model(buffer_type=param_buffer_type, proto_buffer=param_obj, name=name, namespace=namespace) if self.scale_param.method == consts.MINMAXSCALE: cols_scale_value = [] param_dict = param_obj.minmax_scale_param for idx, header_name in enumerate(self.header): if header_name in param_dict: feat_upper = param_dict[header_name].feat_upper feat_lower = param_dict[header_name].feat_lower out_upper = param_dict[header_name].out_upper out_lower = param_dict[header_name].out_lower cols_scale_value.append((feat_lower, feat_upper, out_lower, out_upper)) else: raise ValueError("Can not find the header name {} in model.".format(header_name)) model_scale_value_results = [ cols_scale_value ] elif self.scale_param.method == consts.STANDARDSCALE: mean = [] std = [] param_dict = param_obj.standard_scale_param for idx, header_name in enumerate(self.header): if header_name in param_dict: mean.append(param_dict[header_name].mean) std.append(param_dict[header_name].scale) else: raise ValueError("Can not find the header name {} in model.".format(header_name)) model_scale_value_results = [ mean, std ] else: raise ValueError("Unknown scale method:{}".format(self.scale_param.method)) return model_scale_value_results
def load_model(self, name, namespace): result_obj = feature_selection_param_pb2.FeatureSelectionParam() model_manager.read_model( buffer_type="HeteroFeatureSelection{}.param".format( self.party_name), proto_buffer=result_obj, name=name, namespace=namespace) self.results = list(result_obj.results) left_col_obj = result_obj.final_left_cols self.cols = list(left_col_obj.original_cols) self.left_cols = dict(left_col_obj.left_cols)
def load_model(self, name, namespace): result_obj = feature_selection_param_pb2.FeatureSelectionParam() model_manager.read_model( buffer_type="HeteroFeatureSelectionGuest.param", proto_buffer=result_obj, name=name, namespace=namespace) self.results = list(result_obj.results) if len(self.results) == 0: self.left_cols = -1 else: result_obj = self.results[-1] self.left_cols = list(result_obj.left_cols)
def load_model(self, model_table, model_namespace): LOGGER.info("load model") model_meta = BoostingTreeModelMeta() manager.read_model(buffer_type="HeteroSecureBoostingTreeGuest.meta", proto_buffer=model_meta, name=model_table, namespace=model_namespace) self.set_model_meta(model_meta) model_param = BoostingTreeModelParam() manager.read_model(buffer_type="HeteroSecureBoostingTreeGuest.param", proto_buffer=model_param, name=model_table, namespace=model_namespace) self.set_model_param(model_param)
def load_model(self, name, namespace): result_obj = feature_binning_param_pb2.FeatureBinningParam() return_code = model_manager.read_model(buffer_type="HeteroFeatureBinning{}.param".format(self.party_name), proto_buffer=result_obj, name=name, namespace=namespace) binning_result_obj = dict(result_obj.binning_result.binning_result) host_params = dict(result_obj.host_results) # LOGGER.debug("Party name is :{}".format(self.party_name)) # LOGGER.debug('Loading model, binning_result_obj is : {}'.format(binning_result_obj)) self.binning_result = {} self.host_results = {} self.cols = [] for col_name, iv_attr_obj in binning_result_obj.items(): iv_attr = IVAttributes([], [], [], [], [], []) iv_attr.reconstruct(iv_attr_obj) self.binning_result[col_name] = iv_attr self.cols.append(col_name) for host_name, host_result_obj in host_params.items(): host_result_obj = dict(host_result_obj.binning_result) for col_name, iv_attr_obj in host_result_obj.items(): iv_attr = IVAttributes([], [], [], [], [], []) iv_attr.reconstruct(iv_attr_obj) host_result_obj[col_name] = iv_attr self.host_results[host_name] = host_result_obj return return_code
def load_model(self, name, namespace): result_obj = lr_model_param_pb2.LRModelParam() buffer_type = "{}.param".format(self.class_name) model_manager.read_model(buffer_type=buffer_type, proto_buffer=result_obj, name=name, namespace=namespace) self.header = list(result_obj.header) feature_shape = len(self.header) self.coef_ = np.zeros(feature_shape) weight_dict = dict(result_obj.weight) self.intercept_ = result_obj.intercept for idx, header_name in enumerate(self.header): self.coef_[idx] = weight_dict.get(header_name)
def _load_pipeline(self): buffer_type = "Pipeline" pipeline_obj = pipeline_pb2.Pipeline() pipeline_obj = model_manager.read_model(buffer_type=buffer_type, proto_buffer=pipeline_obj, name=self.workflow_param.model_table, namespace=self.workflow_param.model_namespace) pipeline_obj.node_meta = list(pipeline_obj.node_meta) pipeline_obj.node_param = list(pipeline_obj.node_param) self.pipeline = pipeline_obj
def load_outlier_model(header=None, model_name="Outlier", model_table=None, model_namespace=None): model_meta = OutlierMeta() model_param = OutlierParam() manager.read_model(buffer_type=model_name + ".meta", proto_buffer=model_meta, name=model_table, namespace=model_namespace) manager.read_model(buffer_type=model_name + ".param", proto_buffer=model_param, name=model_table, namespace=model_namespace) outlier_replace = model_meta.is_outlier outlier_replace_method = model_meta.strategy outlier_value = model_meta.outlier_value outlier_replace_value = model_param.outlier_replace_value if outlier_replace: if not outlier_replace_method: outlier_replace_method = None if not outlier_value: outlier_value = None else: outlier_value = list(outlier_value) if outlier_replace_value: outlier_replace_value = [ outlier_replace_value.get(head) for head in header ] else: outlier_replace_value = None else: outlier_replace_method = None outlier_value = None outlier_replace_value = None return outlier_replace, outlier_replace_method, outlier_value, outlier_replace_value
def load_missing_imputer_model(header=None, model_name="Imputer", model_table=None, model_namespace=None): model_meta = ImputerMeta() model_param = ImputerParam() manager.read_model(buffer_type=model_name + ".meta", proto_buffer=model_meta, name=model_table, namespace=model_namespace) manager.read_model(buffer_type=model_name + ".param", proto_buffer=model_param, name=model_table, namespace=model_namespace) missing_fill = model_meta.is_imputer missing_replace_method = model_meta.strategy missing_value = model_meta.missing_value missing_fill_value = model_param.missing_replace_value if missing_fill: if not missing_replace_method: missing_replace_method = None if not missing_value: missing_value = None else: missing_value = list(missing_value) if missing_fill_value: missing_fill_value = [ missing_fill_value.get(head) for head in header ] else: missing_fill_value = None else: missing_replace_method = None missing_value = None missing_fill_value = None return missing_fill, missing_replace_method, missing_value, missing_fill_value
def load_model(self, name, namespace): result_obj = onehot_param_pb2.OneHotParam() return_code = model_manager.read_model(buffer_type='OneHotEncoder.param', proto_buffer=result_obj, name=name, namespace=namespace) self.col_maps = dict(result_obj.col_map) for k, v in self.col_maps.items(): self.col_maps[k] = dict(v.encode_map) return return_code
def load_model(self, name, namespace): result_obj = feature_binning_param_pb2.FeatureBinningParam() model_manager.read_model(buffer_type="HeteroFeatureBinningGuest.param", proto_buffer=result_obj, name=name, namespace=namespace) self.iv_attrs = [] for iv_dict in list(result_obj.iv_result): iv_attr = IVAttributes([], [], [], [], [], [], []) iv_attr.reconstruct(iv_dict) self.iv_attrs.append(iv_attr) self.host_iv_attrs = [] for iv_dict in list(result_obj.host_iv_result): iv_attr = IVAttributes([], [], [], [], [], [], []) iv_attr.reconstruct(iv_dict) self.host_iv_attrs.append(iv_attr) self.cols = list(result_obj.cols)