def __init__(self, tree_param: DecisionTreeModelParam, valid_feature: dict, epoch_idx: int, tree_idx: int, flow_id: int): super(HomoDecisionTreeArbiter, self).__init__(tree_param) self.splitter = Splitter( self.criterion_method, self.criterion_params, self.min_impurity_split, self.min_sample_split, self.min_leaf_node, ) self.transfer_inst = HomoDecisionTreeTransferVariable() """ initializing here """ self.valid_features = valid_feature self.tree_node = [] # start from root node self.tree_node_num = 0 self.cur_layer_node = [] self.runtime_idx = 0 self.sitename = consts.ARBITER self.epoch_idx = epoch_idx self.tree_idx = tree_idx # secure aggregator self.set_flowid(flow_id) self.aggregator = DecisionTreeArbiterAggregator(verbose=False) # stored histogram for faster computation {node_id:histogram_bag} self.stored_histograms = {}
def __init__(self, tree_param: DecisionTreeParam, data_bin=None, bin_split_points: np.array = None, bin_sparse_point=None, g_h=None, valid_feature: dict = None, epoch_idx: int = None, role: str = None, tree_idx: int = None, flow_id: int = None, mode='train'): """ Parameters ---------- tree_param: decision tree parameter object data_bin binned: data instance bin_split_points: data split points bin_sparse_point: sparse data point g_h computed: g val and h val of instances valid_feature: dict points out valid features {valid:true,invalid:false} epoch_idx: current epoch index role: host or guest flow_id: flow id mode: train / predict """ super(HomoDecisionTreeClient, self).__init__(tree_param) self.splitter = Splitter(self.criterion_method, self.criterion_params, self.min_impurity_split, self.min_sample_split, self.min_leaf_node) self.data_bin = data_bin self.g_h = g_h self.bin_split_points = bin_split_points self.bin_sparse_points = bin_sparse_point self.epoch_idx = epoch_idx self.tree_idx = tree_idx self.transfer_inst = HomoDecisionTreeTransferVariable() """ initializing here """ self.valid_features = valid_feature self.tree_node = [] # start from root node self.tree_node_num = 0 self.cur_layer_node = [] self.runtime_idx = 0 self.sitename = consts.GUEST # memory backend self.arr_bin_data = None self.memory_hist_builder = None self.sample_id_arr = None self.bin_num = 0 # secure aggregator, class SecureBoostClientAggregator if mode == 'train': self.role = role self.set_flowid(flow_id) self.aggregator = DecisionTreeClientAggregator(verbose=False) elif mode == 'predict': self.role, self.aggregator = None, None self.check_max_split_nodes() LOGGER.debug('use missing status {} {}'.format(self.use_missing, self.zero_as_missing))
class HomoDecisionTreeArbiter(DecisionTree): def __init__(self, tree_param: DecisionTreeModelParam, valid_feature: dict, epoch_idx: int, tree_idx: int, flow_id: int): super(HomoDecisionTreeArbiter, self).__init__(tree_param) self.splitter = Splitter( self.criterion_method, self.criterion_params, self.min_impurity_split, self.min_sample_split, self.min_leaf_node, ) self.transfer_inst = HomoDecisionTreeTransferVariable() """ initializing here """ self.valid_features = valid_feature self.tree_node = [] # start from root node self.tree_node_num = 0 self.cur_layer_node = [] self.runtime_idx = 0 self.sitename = consts.ARBITER self.epoch_idx = epoch_idx self.tree_idx = tree_idx # secure aggregator self.set_flowid(flow_id) self.aggregator = DecisionTreeArbiterAggregator(verbose=False) # stored histogram for faster computation {node_id:histogram_bag} self.stored_histograms = {} def set_flowid(self, flowid=0): LOGGER.info("set flowid, flowid is {}".format(flowid)) self.transfer_inst.set_flowid(flowid) def sync_node_sample_numbers(self, suffix): cur_layer_node_num = self.transfer_inst.cur_layer_node_num.get( -1, suffix=suffix) for num in cur_layer_node_num[1:]: assert num == cur_layer_node_num[0] return cur_layer_node_num[0] def federated_find_best_split(self, node_histograms, parallel_partitions=10) -> List[SplitInfo]: # node histograms [[HistogramBag,HistogramBag,...],[HistogramBag,HistogramBag,....],..] LOGGER.debug( 'federated finding best splits,histograms from {} guest received'. format(len(node_histograms))) LOGGER.debug('aggregating histograms .....') acc_histogram = node_histograms best_splits = self.splitter.find_split(acc_histogram, self.valid_features, parallel_partitions, self.sitename, self.use_missing, self.zero_as_missing) return best_splits def sync_best_splits(self, split_info, suffix): LOGGER.debug('sending best split points') self.transfer_inst.best_split_points.remote(split_info, idx=-1, suffix=suffix) def sync_local_histogram(self, suffix) -> List[HistogramBag]: LOGGER.debug('get local histograms') node_local_histogram = self.aggregator.aggregate_histogram( suffix=suffix) LOGGER.debug('num of histograms {}'.format(len(node_local_histogram))) return node_local_histogram def histogram_subtraction(self, left_node_histogram, stored_histograms): # histogram subtraction all_histograms = [] for left_hist in left_node_histogram: all_histograms.append(left_hist) # LOGGER.debug('hist id is {}, pid is {}'.format(left_hist.hid, left_hist.p_hid)) # root node hist if left_hist.hid == 0: continue right_hist = stored_histograms[left_hist.p_hid] - left_hist right_hist.hid, right_hist.p_hid = left_hist.hid + 1, right_hist.p_hid all_histograms.append(right_hist) return all_histograms def fit(self): LOGGER.info( 'begin to fit h**o decision tree, epoch {}, tree idx {}'.format( self.epoch_idx, self.tree_idx)) g_sum, h_sum = self.aggregator.aggregate_root_node_info( suffix=('root_node_sync1', self.epoch_idx)) LOGGER.debug('g_sum is {},h_sum is {}'.format(g_sum, h_sum)) self.aggregator.broadcast_root_info(g_sum, h_sum, suffix=('root_node_sync2', self.epoch_idx)) if self.max_split_nodes != 0 and self.max_split_nodes % 2 == 1: self.max_split_nodes += 1 LOGGER.warning( 'an even max_split_nodes value is suggested when using histogram-subtraction, max_split_nodes reset to {}' .format(self.max_split_nodes)) for dep in range(self.max_depth): if dep + 1 == self.max_depth: break LOGGER.debug('at dep {}'.format(dep)) split_info = [] # get cur layer node num: cur_layer_node_num = self.sync_node_sample_numbers( suffix=(dep, self.epoch_idx, self.tree_idx)) LOGGER.debug( '{} nodes to split at this layer'.format(cur_layer_node_num)) layer_stored_hist = {} for batch_id, i in enumerate( range(0, cur_layer_node_num, self.max_split_nodes)): LOGGER.debug('cur batch id is {}'.format(batch_id)) left_node_histogram = self.sync_local_histogram( suffix=(batch_id, dep, self.epoch_idx, self.tree_idx)) all_histograms = self.histogram_subtraction( left_node_histogram, self.stored_histograms) # store histogram for hist in all_histograms: layer_stored_hist[hist.hid] = hist # FIXME stable parallel_partitions best_splits = self.federated_find_best_split( all_histograms, parallel_partitions=10) split_info += best_splits self.stored_histograms = layer_stored_hist self.sync_best_splits(split_info, suffix=(dep, self.epoch_idx)) LOGGER.debug('best_splits_sent') def predict(self, data_inst=None): """ Do nothing """ LOGGER.debug('start predicting')
class HomoDecisionTreeClient(DecisionTree): def __init__(self, tree_param: DecisionTreeParam, data_bin=None, bin_split_points: np.array = None, bin_sparse_point=None, g_h=None, valid_feature: dict = None, epoch_idx: int = None, role: str = None, tree_idx: int = None, flow_id: int = None, mode='train'): """ Parameters ---------- tree_param: decision tree parameter object data_bin binned: data instance bin_split_points: data split points bin_sparse_point: sparse data point g_h computed: g val and h val of instances valid_feature: dict points out valid features {valid:true,invalid:false} epoch_idx: current epoch index role: host or guest flow_id: flow id mode: train / predict """ super(HomoDecisionTreeClient, self).__init__(tree_param) self.splitter = Splitter(self.criterion_method, self.criterion_params, self.min_impurity_split, self.min_sample_split, self.min_leaf_node) self.data_bin = data_bin self.g_h = g_h self.bin_split_points = bin_split_points self.bin_sparse_points = bin_sparse_point self.epoch_idx = epoch_idx self.tree_idx = tree_idx self.transfer_inst = HomoDecisionTreeTransferVariable() """ initializing here """ self.valid_features = valid_feature self.tree_node = [] # start from root node self.tree_node_num = 0 self.cur_layer_node = [] self.runtime_idx = 0 self.sitename = consts.GUEST # memory backend self.arr_bin_data = None self.memory_hist_builder_list = [] self.sample_id_arr = None self.bin_num = 0 # secure aggregator, class SecureBoostClientAggregator if mode == 'train': self.role = role self.set_flowid(flow_id) self.aggregator = DecisionTreeClientAggregator(verbose=False) elif mode == 'predict': self.role, self.aggregator = None, None self.check_max_split_nodes() LOGGER.debug('use missing status {} {}'.format(self.use_missing, self.zero_as_missing)) def set_flowid(self, flowid): LOGGER.info("set flowid, flowid is {}".format(flowid)) self.transfer_inst.set_flowid(flowid) """ Federation functions """ def sync_local_node_histogram(self, acc_histogram: List[HistogramBag], suffix): # sending local histogram self.aggregator.send_histogram(acc_histogram, suffix=suffix) LOGGER.debug('local histogram sent at layer {}'.format(suffix[0])) def sync_cur_layer_node_num(self, node_num, suffix): self.transfer_inst.cur_layer_node_num.remote(node_num, role=consts.ARBITER, idx=-1, suffix=suffix) def sync_best_splits(self, suffix) -> List[SplitInfo]: best_splits = self.transfer_inst.best_split_points.get(idx=0, suffix=suffix) return best_splits """ Computing functions """ def get_node_map(self, nodes: List[Node], left_node_only=True): node_map = {} idx = 0 for node in nodes: if node.id != 0 and (not node.is_left_node and left_node_only): continue node_map[node.id] = idx idx += 1 return node_map def get_grad_hess_sum(self, grad_and_hess_table): LOGGER.info("calculate the sum of grad and hess") grad, hess = grad_and_hess_table.reduce(lambda value1, value2: (value1[ 0] + value2[0], value1[1] + value2[1])) return grad, hess def get_local_histogram(self, cur_to_split: List[Node], g_h, table_with_assign, split_points, sparse_point, valid_feature): LOGGER.info("start to get node histograms") node_map = self.get_node_map(nodes=cur_to_split) histograms = FeatureHistogram.calculate_histogram( table_with_assign, g_h, split_points, sparse_point, valid_feature, node_map, self.use_missing, self.zero_as_missing) hist_bags = [] for hist_list in histograms: hist_bags.append(HistogramBag(hist_list)) return hist_bags def get_left_node_local_histogram(self, cur_nodes: List[Node], tree: List[Node], g_h, table_with_assign, split_points, sparse_point, valid_feature): node_map = self.get_node_map(cur_nodes, left_node_only=True) LOGGER.info("start to get node histograms") histograms = self.hist_computer.calculate_histogram( table_with_assign, g_h, split_points, sparse_point, valid_feature, node_map, self.use_missing, self.zero_as_missing) hist_bags = [] for hist_list in histograms: hist_bags.append(HistogramBag(hist_list)) left_nodes = [] for node in cur_nodes: if node.is_left_node or node.id == 0: left_nodes.append(node) # set histogram id and parent histogram id for node, hist_bag in zip(left_nodes, hist_bags): # LOGGER.debug('node id {}, node parent id {}, cur tree {}'.format(node.id, node.parent_nodeid, len(tree))) hist_bag.hid = node.id hist_bag.p_hid = node.parent_nodeid return hist_bags """ Tree Updating """ def update_tree(self, cur_to_split: List[Node], split_info: List[SplitInfo]): """ update current tree structure ---------- split_info """ LOGGER.debug('updating tree_node, cur layer has {} node'.format( len(cur_to_split))) next_layer_node = [] assert len(cur_to_split) == len(split_info) for idx in range(len(cur_to_split)): sum_grad = cur_to_split[idx].sum_grad sum_hess = cur_to_split[idx].sum_hess if split_info[idx].best_fid is None or split_info[ idx].gain <= self.min_impurity_split + consts.FLOAT_ZERO: cur_to_split[idx].is_leaf = True self.tree_node.append(cur_to_split[idx]) continue cur_to_split[idx].fid = split_info[idx].best_fid cur_to_split[idx].bid = split_info[idx].best_bid cur_to_split[idx].missing_dir = split_info[idx].missing_dir p_id = cur_to_split[idx].id l_id, r_id = self.tree_node_num + 1, self.tree_node_num + 2 cur_to_split[idx].left_nodeid, cur_to_split[ idx].right_nodeid = l_id, r_id self.tree_node_num += 2 l_g, l_h = split_info[idx].sum_grad, split_info[idx].sum_hess # create new left node and new right node left_node = Node(id=l_id, sitename=self.sitename, sum_grad=l_g, sum_hess=l_h, weight=self.splitter.node_weight(l_g, l_h), parent_nodeid=p_id, sibling_nodeid=r_id, is_left_node=True) right_node = Node(id=r_id, sitename=self.sitename, sum_grad=sum_grad - l_g, sum_hess=sum_hess - l_h, weight=self.splitter.node_weight( sum_grad - l_g, sum_hess - l_h), parent_nodeid=p_id, sibling_nodeid=l_id, is_left_node=False) next_layer_node.append(left_node) next_layer_node.append(right_node) self.tree_node.append(cur_to_split[idx]) self.update_feature_importance(split_info[idx], record_site_name=False) return next_layer_node @staticmethod def assign_an_instance(row, tree: List[Node], bin_sparse_point, use_missing, use_zero_as_missing): leaf_status, nodeid = row[1] node = tree[nodeid] if node.is_leaf: return node.id data_inst = row[0] new_layer_nodeid = DecisionTree.go_next_layer( node, data_inst, use_missing, use_zero_as_missing, bin_sparse_point=bin_sparse_point) return 1, new_layer_nodeid def assign_instances_to_new_node(self, table_with_assignment, tree_node: List[Node]): LOGGER.debug('re-assign instance to new nodes') assign_method = functools.partial( self.assign_an_instance, tree=tree_node, bin_sparse_point=self.bin_sparse_points, use_missing=self.use_missing, use_zero_as_missing=self.zero_as_missing) assign_result = table_with_assignment.mapValues(assign_method) leaf_val = assign_result.filter( lambda key, value: isinstance(value, tuple) is False) assign_result = assign_result.subtractByKey(leaf_val) return assign_result, leaf_val def update_instances_node_positions(self, ): return self.data_bin.join(self.inst2node_idx, lambda inst, assignment: (inst, assignment)) """ Pre/Post process """ @staticmethod def get_node_sample_weights(inst2node, tree_node: List[Node]): """ get samples' weights which correspond to its node assignment """ func = functools.partial(lambda inst, nodes: nodes[inst[1]].weight, nodes=tree_node) return inst2node.mapValues(func) def convert_bin_to_real(self): """ convert current bid in tree nodes to real value """ for node in self.tree_node: if not node.is_leaf: node.bid = self.bin_split_points[node.fid][node.bid] def assign_instance_to_root_node(self, data_bin, root_node_id): return data_bin.mapValues(lambda inst: (1, root_node_id)) def init_root_node_and_gh_sum(self): # compute local g_sum and h_sum g_sum, h_sum = self.get_grad_hess_sum(self.g_h) # get aggregated root info self.aggregator.send_local_root_node_info(g_sum, h_sum, suffix=('root_node_sync1', self.epoch_idx)) g_h_dict = self.aggregator.get_aggregated_root_info( suffix=('root_node_sync2', self.epoch_idx)) global_g_sum, global_h_sum = g_h_dict['g_sum'], g_h_dict['h_sum'] # initialize node root_node = Node(id=0, sitename=consts.GUEST, sum_grad=global_g_sum, sum_hess=global_h_sum, weight=self.splitter.node_weight( global_g_sum, global_h_sum)) self.cur_layer_node = [root_node] """ Memory backend functions """ def get_g_h_arr(self): g_, h_ = [], [] for id_, gh in self.g_h.collect(): g_.append(gh[0]) h_.append(gh[1]) g_, h_ = np.array(g_).astype(np.float32), np.array(h_).astype( np.float32) return g_, h_ def init_node2index(self, sample_num): root_sample_idx = np.array([i for i in range(sample_num) ]).astype(np.uint32) return root_sample_idx def init_memory_hist_builder(self, g, h, bin_data, bin_num): if len(g.shape) == 2: # mo case idx_end = g.shape[1] for i in range(0, idx_end): g_arr = np.ascontiguousarray(g[::, i], dtype=np.float32) h_arr = np.ascontiguousarray(h[::, i], dtype=np.float32) hist_builder = HistogramBuilder(bin_data, bin_num, g_arr, h_arr, False) self.memory_hist_builder_list.append(hist_builder) else: self.memory_hist_builder_list.append( HistogramBuilder(bin_data, bin_num, g, h, False)) def sklearn_compute_agg_hist(self, data_indices): hist = [] for memory_hist_builder in self.memory_hist_builder_list: hist_memory_view = memory_hist_builder.compute_histograms_brute( data_indices) hist_arr = np.array(hist_memory_view) g = hist_arr['sum_gradients'].cumsum(axis=1) h = hist_arr['sum_hessians'].cumsum(axis=1) count = hist_arr['count'].cumsum(axis=1) final_hist = [] for feat_idx in range(len(g)): arr = np.array([g[feat_idx], h[feat_idx], count[feat_idx]]).transpose() final_hist.append(arr) hist.append(np.array(final_hist)) # non-mo case, return nd array if len(hist) == 1: return hist[0] # handle mo case, return list multi_dim_g, multi_dim_h, count = None, None, None for dimension_hist in hist: cur_g, cur_h = dimension_hist[::, ::, 0], dimension_hist[::, ::, 1] cur_g = cur_g.reshape(cur_g.shape[0], cur_g.shape[1], 1) cur_h = cur_h.reshape(cur_h.shape[0], cur_h.shape[1], 1) if multi_dim_g is None and multi_dim_h is None: multi_dim_g = cur_g multi_dim_h = cur_h else: multi_dim_g = np.concatenate([multi_dim_g, cur_g], axis=-1) multi_dim_h = np.concatenate([multi_dim_h, cur_h], axis=-1) if count is None: count = dimension_hist[::, ::, 2] # is a slow realization, to improve rs = [] for feat_g, feat_h, feat_c in zip(multi_dim_g, multi_dim_h, count): feat_hist = [[g_arr, h_arr, c] for g_arr, h_arr, c in zip(feat_g, feat_h, feat_c)] rs.append(feat_hist) return rs def assign_arr_inst(self, node, data_arr, data_indices, missing_bin_index=None): # a fast inst assign using memory computing inst = data_arr[data_indices] fid = node.fid bid = node.bid decision = inst[::, fid] <= bid if self.use_missing and missing_bin_index is not None: missing_dir = True if node.missing_dir == -1 else False missing_samples = (inst[::, fid] == missing_bin_index) decision[missing_samples] = missing_dir left_samples = data_indices[decision] right_samples = data_indices[~decision] return left_samples, right_samples """ Fit & Predict """ def memory_fit(self): """ fitting using memory backend """ LOGGER.info('begin to fit h**o decision tree, epoch {}, tree idx {},' 'running on memory backend'.format(self.epoch_idx, self.tree_idx)) self.init_root_node_and_gh_sum() g, h = self.get_g_h_arr() self.init_memory_hist_builder(g, h, self.arr_bin_data, self.bin_num + self.use_missing) # last missing bin root_indices = self.init_node2index(len(self.arr_bin_data)) self.cur_layer_node[0].inst_indices = root_indices # root node tree_height = self.max_depth + 1 # non-leaf node height + 1 layer leaf for dep in range(tree_height): if dep + 1 == tree_height: for node in self.cur_layer_node: node.is_leaf = True self.tree_node.append(node) break self.sync_cur_layer_node_num(len(self.cur_layer_node), suffix=(dep, self.epoch_idx, self.tree_idx)) node_map = self.get_node_map(self.cur_layer_node) node_hists = [] for batch_id, i in enumerate( range(0, len(self.cur_layer_node), self.max_split_nodes)): cur_to_split = self.cur_layer_node[i:i + self.max_split_nodes] for node in cur_to_split: if node.id in node_map: hist = self.sklearn_compute_agg_hist(node.inst_indices) hist_bag = HistogramBag(hist) hist_bag.hid = node.id hist_bag.p_hid = node.parent_nodeid node_hists.append(hist_bag) self.sync_local_node_histogram(node_hists, suffix=(batch_id, dep, self.epoch_idx, self.tree_idx)) node_hists = [] split_info = self.sync_best_splits(suffix=(dep, self.epoch_idx)) new_layer_node = self.update_tree(self.cur_layer_node, split_info) node2inst_idx = [] for node in self.cur_layer_node: if node.is_leaf: continue l, r = self.assign_arr_inst(node, self.arr_bin_data, node.inst_indices, missing_bin_index=self.bin_num) node2inst_idx.append(l) node2inst_idx.append(r) assert len(node2inst_idx) == len(new_layer_node) for node, indices in zip(new_layer_node, node2inst_idx): node.inst_indices = indices self.cur_layer_node = new_layer_node sample_indices, weights = [], [] for node in self.tree_node: if node.is_leaf: sample_indices += list(node.inst_indices) weights += [node.weight] * len(node.inst_indices) else: node.bid = self.bin_split_points[node.fid][int(node.bid)] # post-processing of memory backend fit sample_id = self.sample_id_arr[sample_indices] self.leaf_count = {} for node in self.tree_node: if node.is_leaf: self.leaf_count[node.id] = len(node.inst_indices) LOGGER.debug('leaf count is {}'.format(self.leaf_count)) sample_id_type = type(self.g_h.take(1)[0][0]) self.sample_weights = session.parallelize( [(sample_id_type(id_), weight) for id_, weight in zip(sample_id, weights)], include_key=True, partition=self.data_bin.partitions) def fit(self): """ start to fit """ LOGGER.info('begin to fit h**o decision tree, epoch {}, tree idx {},' 'running on distributed backend'.format( self.epoch_idx, self.tree_idx)) self.init_root_node_and_gh_sum() LOGGER.debug('assign samples to root node') self.inst2node_idx = self.assign_instance_to_root_node( self.data_bin, 0) tree_height = self.max_depth + 1 # non-leaf node height + 1 layer leaf for dep in range(tree_height): if dep + 1 == tree_height: for node in self.cur_layer_node: node.is_leaf = True self.tree_node.append(node) rest_sample_leaf_pos = self.inst2node_idx.mapValues( lambda x: x[1]) if self.sample_leaf_pos is None: self.sample_leaf_pos = rest_sample_leaf_pos else: self.sample_leaf_pos = self.sample_leaf_pos.union( rest_sample_leaf_pos) # stop fitting break LOGGER.debug('start to fit layer {}'.format(dep)) table_with_assignment = self.update_instances_node_positions() # send current layer node number: self.sync_cur_layer_node_num(len(self.cur_layer_node), suffix=(dep, self.epoch_idx, self.tree_idx)) split_info, agg_histograms = [], [] for batch_id, i in enumerate( range(0, len(self.cur_layer_node), self.max_split_nodes)): cur_to_split = self.cur_layer_node[i:i + self.max_split_nodes] node_map = self.get_node_map(nodes=cur_to_split) LOGGER.debug('node map is {}'.format(node_map)) LOGGER.debug( 'computing histogram for batch{} at depth{}'.format( batch_id, dep)) local_histogram = self.get_left_node_local_histogram( cur_nodes=cur_to_split, tree=self.tree_node, g_h=self.g_h, table_with_assign=table_with_assignment, split_points=self.bin_split_points, sparse_point=self.bin_sparse_points, valid_feature=self.valid_features) LOGGER.debug( 'federated finding best splits for batch{} at layer {}'. format(batch_id, dep)) self.sync_local_node_histogram(local_histogram, suffix=(batch_id, dep, self.epoch_idx, self.tree_idx)) agg_histograms += local_histogram split_info = self.sync_best_splits(suffix=(dep, self.epoch_idx)) LOGGER.debug('got best splits from arbiter') new_layer_node = self.update_tree(self.cur_layer_node, split_info) self.cur_layer_node = new_layer_node self.inst2node_idx, leaf_val = self.assign_instances_to_new_node( table_with_assignment, self.tree_node) # record leaf val if self.sample_leaf_pos is None: self.sample_leaf_pos = leaf_val else: self.sample_leaf_pos = self.sample_leaf_pos.union(leaf_val) LOGGER.debug('assigning instance to new nodes done') self.convert_bin_to_real() self.sample_weights_post_process() LOGGER.debug('fitting tree done') def traverse_tree(self, data_inst: Instance, tree: List[Node], use_missing, zero_as_missing): nid = 0 # root node id while True: if tree[nid].is_leaf: return tree[nid].weight nid = DecisionTree.go_next_layer(tree[nid], data_inst, use_missing, zero_as_missing) def predict(self, data_inst): LOGGER.debug('tree start to predict') traverse_tree = functools.partial( self.traverse_tree, tree=self.tree_node, use_missing=self.use_missing, zero_as_missing=self.zero_as_missing, ) predicted_weights = data_inst.mapValues(traverse_tree) return predicted_weights """ Model Outputs """ def get_model_meta(self): model_meta = DecisionTreeModelMeta() model_meta.criterion_meta.CopyFrom( CriterionMeta(criterion_method=self.criterion_method, criterion_param=self.criterion_params)) model_meta.max_depth = self.max_depth model_meta.min_sample_split = self.min_sample_split model_meta.min_impurity_split = self.min_impurity_split model_meta.min_leaf_node = self.min_leaf_node model_meta.use_missing = self.use_missing model_meta.zero_as_missing = self.zero_as_missing return model_meta def set_model_meta(self, model_meta): self.max_depth = model_meta.max_depth self.min_sample_split = model_meta.min_sample_split self.min_impurity_split = model_meta.min_impurity_split self.min_leaf_node = model_meta.min_leaf_node self.criterion_method = model_meta.criterion_meta.criterion_method self.criterion_params = list(model_meta.criterion_meta.criterion_param) self.use_missing = model_meta.use_missing self.zero_as_missing = model_meta.zero_as_missing def get_model_param(self): model_param = DecisionTreeModelParam() for node in self.tree_node: weight, mo_weight = self.mo_weight_extract(node) model_param.tree_.add(id=node.id, sitename=self.role, fid=node.fid, bid=node.bid, weight=weight, is_leaf=node.is_leaf, left_nodeid=node.left_nodeid, right_nodeid=node.right_nodeid, missing_dir=node.missing_dir, mo_weight=mo_weight) model_param.leaf_count.update(self.leaf_count) return model_param def set_model_param(self, model_param): self.tree_node = [] for node_param in model_param.tree_: weight = self.mo_weight_load(node_param) _node = Node(id=node_param.id, sitename=node_param.sitename, fid=node_param.fid, bid=node_param.bid, weight=weight, is_leaf=node_param.is_leaf, left_nodeid=node_param.left_nodeid, right_nodeid=node_param.right_nodeid, missing_dir=node_param.missing_dir) self.tree_node.append(_node) def get_model(self): model_meta = self.get_model_meta() model_param = self.get_model_param() return model_meta, model_param def load_model(self, model_meta=None, model_param=None): LOGGER.info("load tree model") self.set_model_meta(model_meta) self.set_model_param(model_param) def compute_best_splits(self, *args): # not implemented in h**o tree pass def initialize_root_node(self, *args): # not implemented in h**o tree pass
def __init__(self, tree_param: DecisionTreeParam, data_bin=None, bin_split_points: np.array = None, bin_sparse_point=None, g_h=None, valid_feature: dict = None, epoch_idx: int = None, role: str = None, tree_idx: int = None, flow_id: int = None, mode='train'): """ Parameters ---------- tree_param: decision tree parameter object data_bin binned: data instance bin_split_points: data split points bin_sparse_point: sparse data point g_h computed: g val and h val of instances valid_feature: dict points out valid features {valid:true,invalid:false} epoch_idx: current epoch index role: host or guest flow_id: flow id mode: train / predict """ super(HomoDecisionTreeClient, self).__init__(tree_param) self.splitter = Splitter(self.criterion_method, self.criterion_params, self.min_impurity_split, self.min_sample_split, self.min_leaf_node) self.data_bin = data_bin self.g_h = g_h self.bin_split_points = bin_split_points self.bin_sparse_points = bin_sparse_point self.epoch_idx = epoch_idx self.tree_idx = tree_idx # check max_split_nodes if self.max_split_nodes != 0 and self.max_split_nodes % 2 == 1: self.max_split_nodes += 1 LOGGER.warning( 'an even max_split_nodes value is suggested ' 'when using histogram-subtraction, max_split_nodes reset to {}' .format(self.max_split_nodes)) self.transfer_inst = HomoDecisionTreeTransferVariable() """ initializing here """ self.valid_features = valid_feature self.tree_node = [] # start from root node self.tree_node_num = 0 self.cur_layer_node = [] self.runtime_idx = 0 self.sitename = consts.GUEST self.feature_importance = {} # secure aggregator, class SecureBoostClientAggregator if mode == 'train': self.role = role self.set_flowid(flow_id) self.aggregator = DecisionTreeClientAggregator(verbose=False) elif mode == 'predict': self.role, self.aggregator = None, None
class HomoDecisionTreeClient(DecisionTree): def __init__(self, tree_param: DecisionTreeParam, data_bin=None, bin_split_points: np.array = None, bin_sparse_point=None, g_h=None, valid_feature: dict = None, epoch_idx: int = None, role: str = None, tree_idx: int = None, flow_id: int = None, mode='train'): """ Parameters ---------- tree_param: decision tree parameter object data_bin binned: data instance bin_split_points: data split points bin_sparse_point: sparse data point g_h computed: g val and h val of instances valid_feature: dict points out valid features {valid:true,invalid:false} epoch_idx: current epoch index role: host or guest flow_id: flow id mode: train / predict """ super(HomoDecisionTreeClient, self).__init__(tree_param) self.splitter = Splitter(self.criterion_method, self.criterion_params, self.min_impurity_split, self.min_sample_split, self.min_leaf_node) self.data_bin = data_bin self.g_h = g_h self.bin_split_points = bin_split_points self.bin_sparse_points = bin_sparse_point self.epoch_idx = epoch_idx self.tree_idx = tree_idx # check max_split_nodes if self.max_split_nodes != 0 and self.max_split_nodes % 2 == 1: self.max_split_nodes += 1 LOGGER.warning( 'an even max_split_nodes value is suggested ' 'when using histogram-subtraction, max_split_nodes reset to {}' .format(self.max_split_nodes)) self.transfer_inst = HomoDecisionTreeTransferVariable() """ initializing here """ self.valid_features = valid_feature self.tree_node = [] # start from root node self.tree_node_num = 0 self.cur_layer_node = [] self.runtime_idx = 0 self.sitename = consts.GUEST self.feature_importance = {} # secure aggregator, class SecureBoostClientAggregator if mode == 'train': self.role = role self.set_flowid(flow_id) self.aggregator = DecisionTreeClientAggregator(verbose=False) elif mode == 'predict': self.role, self.aggregator = None, None def set_flowid(self, flowid=0): LOGGER.info("set flowid, flowid is {}".format(flowid)) self.transfer_inst.set_flowid(flowid) """ Federation functions """ def sync_local_node_histogram(self, acc_histogram: List[HistogramBag], suffix): # sending local histogram self.aggregator.send_histogram(acc_histogram, suffix=suffix) LOGGER.debug('local histogram sent at layer {}'.format(suffix[0])) def sync_cur_layer_node_num(self, node_num, suffix): self.transfer_inst.cur_layer_node_num.remote(node_num, role=consts.ARBITER, idx=-1, suffix=suffix) def sync_best_splits(self, suffix) -> List[SplitInfo]: best_splits = self.transfer_inst.best_split_points.get(idx=0, suffix=suffix) return best_splits """ Computing functions """ def get_node_map(self, nodes: List[Node], left_node_only=True): node_map = {} idx = 0 for node in nodes: if node.id != 0 and (not node.is_left_node and left_node_only): continue node_map[node.id] = idx idx += 1 return node_map def get_grad_hess_sum(self, grad_and_hess_table): LOGGER.info("calculate the sum of grad and hess") grad, hess = grad_and_hess_table.reduce(lambda value1, value2: (value1[ 0] + value2[0], value1[1] + value2[1])) return grad, hess def get_local_histogram(self, cur_to_split: List[Node], g_h, table_with_assign, split_points, sparse_point, valid_feature): LOGGER.info("start to get node histograms") node_map = self.get_node_map(nodes=cur_to_split) histograms = FeatureHistogram.calculate_histogram( table_with_assign, g_h, split_points, sparse_point, valid_feature, node_map, self.use_missing, self.zero_as_missing) hist_bags = [] for hist_list in histograms: hist_bags.append(HistogramBag(hist_list)) return hist_bags def get_left_node_local_histogram(self, cur_nodes: List[Node], tree: List[Node], g_h, table_with_assign, split_points, sparse_point, valid_feature): node_map = self.get_node_map(cur_nodes, left_node_only=True) LOGGER.info("start to get node histograms") histograms = FeatureHistogram.calculate_histogram( table_with_assign, g_h, split_points, sparse_point, valid_feature, node_map, self.use_missing, self.zero_as_missing) hist_bags = [] for hist_list in histograms: hist_bags.append(HistogramBag(hist_list)) left_nodes = [] for node in cur_nodes: if node.is_left_node or node.id == 0: left_nodes.append(node) # set histogram id and parent histogram id for node, hist_bag in zip(left_nodes, hist_bags): # LOGGER.debug('node id {}, node parent id {}, cur tree {}'.format(node.id, node.parent_nodeid, len(tree))) hist_bag.hid = node.id hist_bag.p_hid = node.parent_nodeid return hist_bags """ Tree Updating """ def update_tree(self, cur_to_split: List[Node], split_info: List[SplitInfo]): """ update current tree structure ---------- split_info """ LOGGER.debug('updating tree_node, cur layer has {} node'.format( len(cur_to_split))) next_layer_node = [] assert len(cur_to_split) == len(split_info) for idx in range(len(cur_to_split)): sum_grad = cur_to_split[idx].sum_grad sum_hess = cur_to_split[idx].sum_hess if split_info[idx].best_fid is None or split_info[ idx].gain <= self.min_impurity_split + consts.FLOAT_ZERO: cur_to_split[idx].is_leaf = True self.tree_node.append(cur_to_split[idx]) continue cur_to_split[idx].fid = split_info[idx].best_fid cur_to_split[idx].bid = split_info[idx].best_bid cur_to_split[idx].missing_dir = split_info[idx].missing_dir p_id = cur_to_split[idx].id l_id, r_id = self.tree_node_num + 1, self.tree_node_num + 2 cur_to_split[idx].left_nodeid, cur_to_split[ idx].right_nodeid = l_id, r_id self.tree_node_num += 2 l_g, l_h = split_info[idx].sum_grad, split_info[idx].sum_hess # create new left node and new right node left_node = Node(id=l_id, sitename=self.sitename, sum_grad=l_g, sum_hess=l_h, weight=self.splitter.node_weight(l_g, l_h), parent_nodeid=p_id, sibling_nodeid=r_id, is_left_node=True) right_node = Node(id=r_id, sitename=self.sitename, sum_grad=sum_grad - l_g, sum_hess=sum_hess - l_h, weight=self.splitter.node_weight( sum_grad - l_g, sum_hess - l_h), parent_nodeid=p_id, sibling_nodeid=l_id, is_left_node=False) next_layer_node.append(left_node) next_layer_node.append(right_node) self.tree_node.append(cur_to_split[idx]) self.update_feature_importance(split_info[idx], record_site_name=False) return next_layer_node @staticmethod def assign_a_instance(row, tree: List[Node], bin_sparse_point, use_missing, use_zero_as_missing): leaf_status, nodeid = row[1] node = tree[nodeid] if node.is_leaf: return node.weight fid = node.fid bid = node.bid missing_dir = node.missing_dir missing_val = False if use_zero_as_missing: if row[0].features.get_data(fid, None) is None or \ row[0].features.get_data(fid) == NoneType(): missing_val = True elif use_missing and row[0].features.get_data(fid) == NoneType(): missing_val = True if missing_val: if missing_dir == 1: return 1, tree[nodeid].right_nodeid else: return 1, tree[nodeid].left_nodeid else: if row[0].features.get_data(fid, bin_sparse_point[fid]) <= bid: return 1, tree[nodeid].left_nodeid else: return 1, tree[nodeid].right_nodeid def assign_instances_to_new_node(self, table_with_assignment, tree_node: List[Node]): LOGGER.debug('re-assign instance to new nodes') assign_method = functools.partial( self.assign_a_instance, tree=tree_node, bin_sparse_point=self.bin_sparse_points, use_missing=self.use_missing, use_zero_as_missing=self.zero_as_missing) assign_result = table_with_assignment.mapValues(assign_method) leaf_val = assign_result.filter( lambda key, value: isinstance(value, tuple) is False) assign_result = assign_result.subtractByKey(leaf_val) return assign_result, leaf_val def update_instances_node_positions(self, ): return self.data_bin.join(self.inst2node_idx, lambda inst, assignment: (inst, assignment)) """ Pre/Post process """ @staticmethod def get_node_sample_weights(inst2node, tree_node: List[Node]): """ get samples' weights which correspond to its node assignment """ func = functools.partial(lambda inst, nodes: nodes[inst[1]].weight, nodes=tree_node) return inst2node.mapValues(func) def get_feature_importance(self): return self.feature_importance def convert_bin_to_real(self): """ convert current bid in tree nodes to real value """ for node in self.tree_node: if not node.is_leaf: node.bid = self.bin_split_points[node.fid][node.bid] def assign_instance_to_root_node(self, data_bin, root_node_id): return data_bin.mapValues(lambda inst: (1, root_node_id)) """ Fit & Predict """ def fit(self): """ start to fit """ LOGGER.info( 'begin to fit h**o decision tree, epoch {}, tree idx {}'.format( self.epoch_idx, self.tree_idx)) # compute local g_sum and h_sum g_sum, h_sum = self.get_grad_hess_sum(self.g_h) # get aggregated root info self.aggregator.send_local_root_node_info(g_sum, h_sum, suffix=('root_node_sync1', self.epoch_idx)) g_h_dict = self.aggregator.get_aggregated_root_info( suffix=('root_node_sync2', self.epoch_idx)) global_g_sum, global_h_sum = g_h_dict['g_sum'], g_h_dict['h_sum'] # initialize node root_node = Node(id=0, sitename=consts.GUEST, sum_grad=global_g_sum, sum_hess=global_h_sum, weight=self.splitter.node_weight( global_g_sum, global_h_sum)) self.cur_layer_node = [root_node] LOGGER.debug('assign samples to root node') self.inst2node_idx = self.assign_instance_to_root_node( self.data_bin, 0) tree_height = self.max_depth + 1 # non-leaf node height + 1 layer leaf for dep in range(tree_height): if dep + 1 == tree_height: for node in self.cur_layer_node: node.is_leaf = True self.tree_node.append(node) rest_sample_weights = self.get_node_sample_weights( self.inst2node_idx, self.tree_node) if self.sample_weights is None: self.sample_weights = rest_sample_weights else: self.sample_weights = self.sample_weights.union( rest_sample_weights) # stop fitting break LOGGER.debug('start to fit layer {}'.format(dep)) table_with_assignment = self.update_instances_node_positions() # send current layer node number: self.sync_cur_layer_node_num(len(self.cur_layer_node), suffix=(dep, self.epoch_idx, self.tree_idx)) split_info, agg_histograms = [], [] for batch_id, i in enumerate( range(0, len(self.cur_layer_node), self.max_split_nodes)): cur_to_split = self.cur_layer_node[i:i + self.max_split_nodes] node_map = self.get_node_map(nodes=cur_to_split) LOGGER.debug('node map is {}'.format(node_map)) LOGGER.debug( 'computing histogram for batch{} at depth{}'.format( batch_id, dep)) local_histogram = self.get_left_node_local_histogram( cur_nodes=cur_to_split, tree=self.tree_node, g_h=self.g_h, table_with_assign=table_with_assignment, split_points=self.bin_split_points, sparse_point=self.bin_sparse_points, valid_feature=self.valid_features) LOGGER.debug( 'federated finding best splits for batch{} at layer {}'. format(batch_id, dep)) self.sync_local_node_histogram(local_histogram, suffix=(batch_id, dep, self.epoch_idx, self.tree_idx)) agg_histograms += local_histogram split_info = self.sync_best_splits(suffix=(dep, self.epoch_idx)) LOGGER.debug('got best splits from arbiter') new_layer_node = self.update_tree(self.cur_layer_node, split_info) self.cur_layer_node = new_layer_node self.inst2node_idx, leaf_val = self.assign_instances_to_new_node( table_with_assignment, self.tree_node) # record leaf val if self.sample_weights is None: self.sample_weights = leaf_val else: self.sample_weights = self.sample_weights.union(leaf_val) LOGGER.debug('assigning instance to new nodes done') self.convert_bin_to_real() LOGGER.debug('fitting tree done') def traverse_tree(self, data_inst: Instance, tree: List[Node], use_missing, zero_as_missing): nid = 0 # root node id while True: if tree[nid].is_leaf: return tree[nid].weight cur_node = tree[nid] fid, bid = cur_node.fid, cur_node.bid missing_dir = cur_node.missing_dir if use_missing and zero_as_missing: if data_inst.features.get_data(fid) == NoneType( ) or data_inst.features.get_data(fid, None) is None: nid = tree[nid].right_nodeid if missing_dir == 1 else tree[ nid].left_nodeid elif data_inst.features.get_data( fid) <= bid + consts.FLOAT_ZERO: nid = tree[nid].left_nodeid else: nid = tree[nid].right_nodeid elif data_inst.features.get_data(fid) == NoneType(): nid = tree[nid].right_nodeid if missing_dir == 1 else tree[ nid].left_nodeid elif data_inst.features.get_data(fid, 0) <= bid + consts.FLOAT_ZERO: nid = tree[nid].left_nodeid else: nid = tree[nid].right_nodeid def predict(self, data_inst): LOGGER.debug('tree start to predict') traverse_tree = functools.partial( self.traverse_tree, tree=self.tree_node, use_missing=self.use_missing, zero_as_missing=self.zero_as_missing, ) predicted_weights = data_inst.mapValues(traverse_tree) return predicted_weights """ Model Outputs """ def get_model_meta(self): model_meta = DecisionTreeModelMeta() model_meta.criterion_meta.CopyFrom( CriterionMeta(criterion_method=self.criterion_method, criterion_param=self.criterion_params)) model_meta.max_depth = self.max_depth model_meta.min_sample_split = self.min_sample_split model_meta.min_impurity_split = self.min_impurity_split model_meta.min_leaf_node = self.min_leaf_node model_meta.use_missing = self.use_missing model_meta.zero_as_missing = self.zero_as_missing return model_meta def set_model_meta(self, model_meta): self.max_depth = model_meta.max_depth self.min_sample_split = model_meta.min_sample_split self.min_impurity_split = model_meta.min_impurity_split self.min_leaf_node = model_meta.min_leaf_node self.criterion_method = model_meta.criterion_meta.criterion_method self.criterion_params = list(model_meta.criterion_meta.criterion_param) self.use_missing = model_meta.use_missing self.zero_as_missing = model_meta.zero_as_missing def get_model_param(self): model_param = DecisionTreeModelParam() for node in self.tree_node: model_param.tree_.add(id=node.id, sitename=self.role, fid=node.fid, bid=node.bid, weight=node.weight, is_leaf=node.is_leaf, left_nodeid=node.left_nodeid, right_nodeid=node.right_nodeid, missing_dir=node.missing_dir) LOGGER.debug('output tree: epoch_idx:{} tree_idx:{}'.format( self.epoch_idx, self.tree_idx)) return model_param def set_model_param(self, model_param): self.tree_node = [] for node_param in model_param.tree_: _node = Node(id=node_param.id, sitename=node_param.sitename, fid=node_param.fid, bid=node_param.bid, weight=node_param.weight, is_leaf=node_param.is_leaf, left_nodeid=node_param.left_nodeid, right_nodeid=node_param.right_nodeid, missing_dir=node_param.missing_dir) self.tree_node.append(_node) def get_model(self): model_meta = self.get_model_meta() model_param = self.get_model_param() return model_meta, model_param def load_model(self, model_meta=None, model_param=None): LOGGER.info("load tree model") self.set_model_meta(model_meta) self.set_model_param(model_param) def compute_best_splits(self, *args): # not implemented in h**o tree pass def initialize_root_node(self, *args): # not implemented in h**o tree pass