def client_align_labels(self, data_inst): local_labels = data_inst.map(lambda k, v: [k, {v.label}]).reduce( lambda x, y: x | y) _, self._label_align_mapping = HomoLabelEncoderClient().label_alignment( local_labels) num_classes = len(self._label_align_mapping) if self.config_type == "pytorch": for layer in reversed(self.nn_define): if layer["layer"] == "Linear": output_dim = layer["config"][1] if output_dim == 1 and num_classes == 2: return layer["config"][1] = num_classes return if self.config_type == "nn": for layer in reversed(self.nn_define): if layer["layer"] == "Dense": output_dim = layer.get("units", None) if output_dim == 1 and num_classes == 2: return layer["units"] = num_classes return if self.config_type == "keras": layers = self.nn_define["config"]["layers"] for layer in reversed(layers): if layer["class_name"] == "Dense": output_dim = layer["config"].get("units", None) if output_dim == 1 and num_classes == 2: return layer["config"]["units"] = num_classes return
def _client_check_data(self, data_instances): self._abnormal_detection(data_instances) self.check_abnormal_values(data_instances) self.init_schema(data_instances) num_classes, classes_ = ClassifyLabelChecker.validate_label(data_instances) aligned_label, new_label_mapping = HomoLabelEncoderClient().label_alignment(classes_) if len(aligned_label) > 2: raise ValueError("H**o LR support binary classification only now") elif len(aligned_label) <= 1: raise ValueError("Number of classes should be equal to 2")
def __init__( self, data_instances: CTableABC, expected_label_type=np.float32, label_align_mapping=None, **kwargs, ): # partition self.partitions = data_instances.partitions # size self.size = data_instances.count() if self.size <= 0: raise ValueError("num of instances is 0") # alignment labels if label_align_mapping is None: labels = data_instances.applyPartitions( lambda it: {item[1].label for item in it}).reduce( lambda x, y: set.union(x, y)) _, label_align_mapping = HomoLabelEncoderClient().label_alignment( labels) LOGGER.debug(f"label mapping: {label_align_mapping}") self._label_align_mapping = label_align_mapping # shape self.x_shape = data_instances.first()[1].features.shape self.x = np.zeros((self.size, *self.x_shape), dtype=np.float32) self.y = np.zeros((self.size, ), dtype=expected_label_type) self._keys = [] index = 0 for key, instance in data_instances.collect(): self._keys.append(key) self.x[index] = instance.features self.y[index] = label_align_mapping[instance.label] index += 1 self._num_labels = len(label_align_mapping) self._num_features = self.x_shape[0]
def __init__(self, root, is_train=True, expected_label_type=np.float32, **kwargs): # fake alignment if is_train: HomoLabelEncoderClient().label_alignment(["fake"]) # load data with open(os.path.join(root, "config.yaml")) as f: config = yaml.safe_load(f) if config["type"] == "vision": # read filenames with open(os.path.join(root, "filenames")) as f: file_names = [filename.strip() for filename in f] # read inputs if config["inputs"]["type"] != "images": raise TypeError("inputs type of vision type should be images") input_ext = config["inputs"]["ext"] self.images = [ os.path.join(root, "images", f"{x}.{input_ext}") for x in file_names ] self._PIL_mode = config["inputs"].get("PIL_mode", "L") # read targets if config["targets"]["type"] == "images": target_ext = config["targets"]["ext"] self.targets = [ os.path.join(root, "targets", f"{x}.{target_ext}") for x in file_names ] self.targets_is_image = True elif config["targets"]["type"] == "integer": with open(os.path.join(root, "targets")) as f: targets_mapping = {} for line in f: filename, target = line.split(",") targets_mapping[ filename.strip()] = expected_label_type( target.strip()) self.targets = [ targets_mapping[filename] for filename in file_names ] self.targets_is_image = False self._keys = file_names else: raise TypeError(f"{config['type']}") assert len(self.images) == len(self.targets) transform = torchvision.transforms.Compose([ # torchvision.transforms.Resize((100, 100)), torchvision.transforms.ToTensor(), ]) target_transform = None if self.targets_is_image: target_transform = transform super(VisionDataSet, self).__init__( root, transform=transform, target_transform=target_transform, )
def fit(self, data_inst, validate_data=None): # binning data_inst = self.data_alignment(data_inst) self.data_bin, self.bin_split_points, self.bin_sparse_points = self.federated_binning( data_inst) # fid mapping self.feature_name_fid_mapping = self.gen_feature_fid_mapping( data_inst.schema) # set feature_num self.feature_num = self.bin_split_points.shape[0] # sync feature num self.sync_feature_num() # initialize validation strategy self.validation_strategy = self.init_validation_strategy( train_data=data_inst, validate_data=validate_data, ) # check labels local_classes = self.check_label(self.data_bin) # sync label class and set y if self.task_type == consts.CLASSIFICATION: aligned_label, new_label_mapping = HomoLabelEncoderClient( ).label_alignment(local_classes) self.classes_ = aligned_label self.check_label_starts_from_zero(self.classes_) # set labels self.num_classes = len(new_label_mapping) LOGGER.info('aligned labels are {}, num_classes is {}'.format( aligned_label, self.num_classes)) self.y = self.data_bin.mapValues( lambda instance: new_label_mapping[instance.label]) # set tree dimension self.booster_dim = self.num_classes if self.num_classes > 2 else 1 else: self.y = self.data_bin.mapValues(lambda instance: instance.label) # set loss function self.loss = self.get_loss_function() # set y_hat_val self.y_hat, self.init_score = self.get_init_score( self.y, self.num_classes) LOGGER.info('begin to fit a boosting tree') for epoch_idx in range(self.boosting_round): LOGGER.info('cur epoch idx is {}'.format(epoch_idx)) for class_idx in range(self.booster_dim): # fit a booster model = self.fit_a_booster(epoch_idx, class_idx) booster_meta, booster_param = model.get_model() if booster_meta is not None and booster_param is not None: self.booster_meta = booster_meta self.boosting_model_list.append(booster_param) # update predict score cur_sample_weights = model.get_sample_weights() self.y_hat = self.get_new_predict_score(self.y_hat, cur_sample_weights, dim=class_idx) local_loss = self.compute_loss(self.y_hat, self.y) self.aggregator.send_local_loss(local_loss, self.data_bin.count(), suffix=(epoch_idx, )) if self.validation_strategy: self.validation_strategy.validate(self, epoch_idx) # check stop flag if n_iter_no_change is True if self.n_iter_no_change: should_stop = self.aggregator.get_converge_status( suffix=(str(epoch_idx), )) if should_stop: LOGGER.info('n_iter_no_change stop triggered') break self.set_summary(self.generate_summary())
def fit(self, data_inst, validate_data=None): # init federation obj self.aggregator = HomoBoostClientAggregator() self.binning_obj = HomoFeatureBinningClient() # binning self.data_preporcess(data_inst) # fid mapping and warm start check if not self.is_warm_start: self.feature_name_fid_mapping = self.gen_feature_fid_mapping( data_inst.schema) else: self.feat_name_check(data_inst, self.feature_name_fid_mapping) # set feature_num self.feature_num = self.bin_split_points.shape[0] # sync feature num self.sync_feature_num() # initialize validation strategy self.callback_list.on_train_begin(data_inst, validate_data) # check labels local_classes = self.check_label(self.data_bin) # set start round self.start_round = len(self.boosting_model_list) // self.booster_dim # sync label class and set y if self.task_type == consts.CLASSIFICATION: aligned_label, new_label_mapping = HomoLabelEncoderClient( ).label_alignment(local_classes) if self.is_warm_start: assert set(aligned_label) == set(self.classes_), 'warm start label alignment failed, differences: {}'. \ format(set(aligned_label).symmetric_difference(set(self.classes_))) self.classes_ = aligned_label self.check_label_starts_from_zero(self.classes_) # set labels self.num_classes = len(new_label_mapping) LOGGER.info('aligned labels are {}, num_classes is {}'.format( aligned_label, self.num_classes)) self.y = self.data_bin.mapValues( lambda instance: new_label_mapping[instance.label]) # set tree dimension self.booster_dim = self.num_classes if self.num_classes > 2 else 1 else: self.y = self.data_bin.mapValues(lambda instance: instance.label) # set loss function self.loss = self.get_loss_function() # set y_hat_val, if warm start predict cur samples if self.is_warm_start: self.y_hat = self.predict(data_inst, ret_format='raw') self.boosting_round += self.start_round self.callback_warm_start_init_iter(self.start_round) else: self.y_hat, self.init_score = self.get_init_score( self.y, self.num_classes) # sync start round and end round self.sync_start_round_and_end_round() self.preprocess() LOGGER.info('begin to fit a boosting tree') for epoch_idx in range(self.start_round, self.boosting_round): LOGGER.info('cur epoch idx is {}'.format(epoch_idx)) self.callback_list.on_epoch_begin(epoch_idx) for class_idx in range(self.booster_dim): # fit a booster model = self.fit_a_learner(epoch_idx, class_idx) booster_meta, booster_param = model.get_model() if booster_meta is not None and booster_param is not None: self.booster_meta = booster_meta self.boosting_model_list.append(booster_param) # update predict score cur_sample_weights = model.get_sample_weights() self.y_hat = self.get_new_predict_score(self.y_hat, cur_sample_weights, dim=class_idx) local_loss = self.compute_loss(self.y_hat, self.y) self.aggregator.send_local_loss(local_loss, self.data_bin.count(), suffix=(epoch_idx, )) validation_strategy = self.callback_list.get_validation_strategy() if validation_strategy: validation_strategy.set_precomputed_train_scores( self.score_to_predict_result(data_inst, self.y_hat)) self.callback_list.on_epoch_end(epoch_idx) # check stop flag if n_iter_no_change is True if self.n_iter_no_change: should_stop = self.aggregator.get_converge_status( suffix=(str(epoch_idx), )) if should_stop: LOGGER.info('n_iter_no_change stop triggered') break self.postprocess() self.callback_list.on_train_end() self.set_summary(self.generate_summary())