def encrypt(self, input_data): """ Encrypt data according to different mode Parameters ---------- input_data: DTable Returns ------- new_data: DTable, encrypted result of input_data """ if self.prev_data is None or self.mode == "strict" \ or (self.mode == "balance" and self.gen_random_number() <= self.re_encrypted_rate + consts.FLOAT_ZERO): new_data = input_data.mapValues(self.encrypt_row) else: diff_data = input_data.join(self.prev_data, self.get_differance) new_data = diff_data.join(self.prev_encrypted_data, self.add_differance) """temporary code, clear unused table begin""" rubbish_list = [diff_data] rubbish_clear(rubbish_list) """temporary code, clear unused table end""" # new_data = input_data.join(self.prev_data, self.get_differance).join(self.prev_encrypted_data, self.add_differance) """temporary code, clear unused table begin""" rubbish_list = [self.prev_data, self.prev_encrypted_data] rubbish_clear(rubbish_list) """temporary code, clear unused table end""" self.prev_data = input_data.mapValues(lambda val: val) self.prev_encrypted_data = new_data.mapValues(lambda val: val) return new_data
def compute_forward(self, data_instances, coef_, intercept_, batch_index=-1): """ Compute W * X + b and (W * X + b)^2, where X is the input data, W is the coefficient of lr, and b is the interception Parameters ---------- data_instances: DTable of Instance, input data coef_: list, coefficient of lr intercept_: float, the interception of lr """ wx = self.compute_wx(data_instances, coef_, intercept_) en_wx = self.encrypted_calculator[batch_index].encrypt(wx) wx_square = wx.mapValues(lambda v: np.square(v)) en_wx_square = self.encrypted_calculator[batch_index].encrypt( wx_square) host_forward = en_wx.join(en_wx_square, lambda wx, wx_square: (wx, wx_square)) # temporary resource recovery and will be removed in the future rubbish_list = [wx, en_wx, wx_square, en_wx_square] rubbish_clear(rubbish_list) return host_forward
def compute_gradient_and_loss(self, data_instance, fore_gradient, encrypted_ee, en_ee_square): """ Compute gradient and loss Parameters ---------- data_instance: DTable, input data fore_gradient:DTable, fore_gradient = (1/2*y*e_i*e_j-1)*1/(2y) encrypted_ee: DTable, encrypted ee encrypted_sum_squre: DTable, encrypted ee^2 Return ------ DTable the hetero-lr gradient and loss """ # compute gradient gradients = self.compute_gradient(data_instance, fore_gradient) # compute loss half_yee = encrypted_ee.join(data_instance, lambda ee, d: 0.5 * ee * int(d.label)) half_yee_join_en_ee_square = half_yee.join(en_ee_square, lambda yz, ez: (yz, ez)) f = functools.partial(self.__compute_loss) loss_partition = half_yee_join_en_ee_square.mapPartitions(f).reduce(lambda x, y: x + y) loss = loss_partition[0] / loss_partition[1] # temporary resource recovery and will be removed in the future rubbish_list = [half_yee, half_yee_join_en_ee_square] rubbish_clear(rubbish_list) return gradients, loss
def compute_gradient(self, data_instance, fore_gradient, fit_intercept): """ Compute hetero-lr gradient Parameters ---------- data_instance: DTable, input data fore_gradient: DTable, fore_gradient = (1/2*ywx-1)*1/2y fit_intercept: bool, if hetero-lr has interception or not Returns ---------- DTable the hetero-lr's gradient """ feat_join_grad = data_instance.join(fore_gradient, lambda d, g: (d.features, g)) f = functools.partial(self.__compute_gradient, fit_intercept=fit_intercept) gradient_partition = feat_join_grad.mapPartitions(f).reduce( lambda x, y: x + y) gradient = gradient_partition[:-1] / gradient_partition[-1] for i in range(len(gradient)): if not isinstance(gradient[i], PaillierEncryptedNumber): gradient[i] = self.encrypt_operator.encrypt(gradient[i]) # temporary resource recovery and will be removed in the future rubbish_list = [feat_join_grad] rubbish_clear(rubbish_list) return gradient
def compute_gradient_and_loss(self, data_instance, fore_gradient, encrypted_wx, en_sum_wx_square, fit_intercept): """ Compute gradient and loss Parameters ---------- data_instance: DTable, input data fore_gradient: DTable, fore_gradient = (1/2*ywx-1)*1/2y encrypted_wx: DTable, encrypted wx en_sum_wx_square: DTable, encrypted wx^2 fit_intercept: bool, if hetero-lr has interception or not Returns ---------- DTable the hetero-lr gradient and loss """ # compute gradient gradient = self.compute_gradient(data_instance, fore_gradient, fit_intercept) # compute and loss half_ywx = encrypted_wx.join(data_instance, lambda wx, d: 0.5 * wx * int(d.label)) half_ywx_join_en_sum_wx_square = half_ywx.join(en_sum_wx_square, lambda yz, ez: (yz, ez)) f = functools.partial(self.__compute_loss) loss_partition = half_ywx_join_en_sum_wx_square.mapPartitions(f).reduce(lambda x, y: x + y) loss = loss_partition[0] / loss_partition[1] # temporary resource recovery and will be removed in the future rubbish_list = [half_ywx, half_ywx_join_en_sum_wx_square] rubbish_clear(rubbish_list) return gradient, loss
def compute_forward(self, data_instance, embedding_, node2id, batch_index=-1): """ Compute e_guest, where e_guest is the node embedding which stays on guest """ e = self.compute_embedding(data_instance, embedding_, node2id) en_e = self.encrypted_calculator[batch_index].encrypt(e) forward = en_e.join(e, lambda en_e, e: (en_e, e)) rubbish_list = [e, en_e] rubbish_clear(rubbish_list) return forward
def fit(self, data_instances, node2id, local_instances=None, common_nodes=None): """ Train node embedding for role guest Parameters ---------- data_instances: DTable of target node and label, input data node2id: a dict which can map node name to id """ LOGGER.info("samples number:{}".format(data_instances.count())) LOGGER.info("Enter network embedding procedure:") self.n_node = len(node2id) LOGGER.info("Bank A has {} nodes".format(self.n_node)) data_instances = data_instances.mapValues(HeteroNEGuest.load_data) LOGGER.info("Transform input data to train instance") public_key = federation.get( name=self.transfer_variable.paillier_pubkey.name, tag=self.transfer_variable.generate_transferid( self.transfer_variable.paillier_pubkey), idx=0) LOGGER.info("Get public_key from arbiter:{}".format(public_key)) self.encrypt_operator.set_public_key(public_key) # hetero network embedding LOGGER.info("Generate mini-batch from input data") mini_batch_obj = MiniBatch(data_instances, batch_size=self.batch_size) batch_num = mini_batch_obj.batch_nums LOGGER.info("samples number:{}".format(data_instances.count())) if self.batch_size == -1: LOGGER.info( "batch size is -1, set it to the number of data in data_instances" ) self.batch_size = data_instances.count() ############## # horizontal federated learning LOGGER.info("Generate mini-batch for local instances in guest") mini_batch_obj_local = MiniBatch(local_instances, batch_size=self.batch_size) local_batch_num = mini_batch_obj_local.batch_nums common_node_instances = eggroll.parallelize( ((node, node) for node in common_nodes), include_key=True, name='common_nodes') ############## batch_info = {'batch_size': self.batch_size, "batch_num": batch_num} LOGGER.info("batch_info:{}".format(batch_info)) federation.remote(batch_info, name=self.transfer_variable.batch_info.name, tag=self.transfer_variable.generate_transferid( self.transfer_variable.batch_info), role=consts.HOST, idx=0) LOGGER.info("Remote batch_info to Host") federation.remote(batch_info, name=self.transfer_variable.batch_info.name, tag=self.transfer_variable.generate_transferid( self.transfer_variable.batch_info), role=consts.ARBITER, idx=0) LOGGER.info("Remote batch_info to Arbiter") self.encrypted_calculator = [ EncryptModeCalculator( self.encrypt_operator, self.encrypted_mode_calculator_param.mode, self.encrypted_mode_calculator_param.re_encrypted_rate) for _ in range(batch_num) ] LOGGER.info("Start initialize model.") self.embedding_ = self.initializer.init_model((self.n_node, self.dim), self.init_param_obj) LOGGER.info("Embedding shape={}".format(self.embedding_.shape)) is_send_all_batch_index = False self.n_iter_ = 0 index_data_inst_map = {} while self.n_iter_ < self.max_iter: LOGGER.info("iter:{}".format(self.n_iter_)) ################# local_batch_data_generator = mini_batch_obj_local.mini_batch_data_generator( ) total_loss = 0 local_batch_num = 0 LOGGER.info("Enter the horizontally federated learning procedure:") for local_batch_data in local_batch_data_generator: n = local_batch_data.count() #LOGGER.info("Local batch data count:{}".format(n)) E_Y = self.compute_local_embedding(local_batch_data, self.embedding_, node2id) local_grads_e1, local_grads_e2, local_loss = self.local_gradient_operator.compute( E_Y, 'E_1') local_grads_e1 = local_grads_e1.mapValues( lambda g: self.local_optimizer.apply_gradients(g / n)) local_grads_e2 = local_grads_e2.mapValues( lambda g: self.local_optimizer.apply_gradients(g / n)) e1id_join_grads = local_batch_data.join( local_grads_e1, lambda v, g: (node2id[v[0]], g)) e2id_join_grads = local_batch_data.join( local_grads_e2, lambda v, g: (node2id[v[1]], g)) self.update_model(e1id_join_grads) self.update_model(e2id_join_grads) local_loss = local_loss / n local_batch_num += 1 total_loss += local_loss #LOGGER.info("gradient count:{}".format(e1id_join_grads.count())) guest_common_embedding = common_node_instances.mapValues( lambda node: self.embedding_[node2id[node]]) federation.remote( guest_common_embedding, name=self.transfer_variable.guest_common_embedding.name, tag=self.transfer_variable.generate_transferid( self.transfer_variable.guest_common_embedding, self.n_iter_, 0), role=consts.ARBITER, idx=0) LOGGER.info("Remote the embedding of common node to arbiter!") common_embedding = federation.get( name=self.transfer_variable.common_embedding.name, tag=self.transfer_variable.generate_transferid( self.transfer_variable.common_embedding, self.n_iter_, 0), idx=0) LOGGER.info( "Get the aggregated embedding of common node from arbiter!") self.update_common_nodes(common_embedding, common_nodes, node2id) total_loss /= local_batch_num LOGGER.info( "Iter {}, horizontally feaderated learning loss: {}".format( self.n_iter_, total_loss)) ################# # verticallly feaderated learning # each iter will get the same batch_data_generator LOGGER.info("Enter the vertically federated learning:") batch_data_generator = mini_batch_obj.mini_batch_data_generator( result='index') batch_index = 0 for batch_data_index in batch_data_generator: LOGGER.info("batch:{}".format(batch_index)) # only need to send one times if not is_send_all_batch_index: LOGGER.info("remote mini-batch index to Host") federation.remote( batch_data_index, name=self.transfer_variable.batch_data_index.name, tag=self.transfer_variable.generate_transferid( self.transfer_variable.batch_data_index, self.n_iter_, batch_index), role=consts.HOST, idx=0) if batch_index >= mini_batch_obj.batch_nums - 1: is_send_all_batch_index = True # in order to avoid joining in next iteration # Get mini-batch train data if len(index_data_inst_map) < batch_num: batch_data_inst = data_instances.join( batch_data_index, lambda data_inst, index: data_inst) index_data_inst_map[batch_index] = batch_data_inst else: batch_data_inst = index_data_inst_map[batch_index] # For inductive learning: transform node attributes to node embedding # self.transform(batch_data_inst) self.guest_forward = self.compute_forward( batch_data_inst, self.embedding_, node2id, batch_index) host_forward = federation.get( name=self.transfer_variable.host_forward_dict.name, tag=self.transfer_variable.generate_transferid( self.transfer_variable.host_forward_dict, self.n_iter_, batch_index), idx=0) LOGGER.info("Get host_forward from host") aggregate_forward_res = self.aggregate_forward(host_forward) en_aggregate_ee = aggregate_forward_res.mapValues( lambda v: v[0]) en_aggregate_ee_square = aggregate_forward_res.mapValues( lambda v: v[1]) # compute [[d]] if self.gradient_operator is None: self.gradient_operator = HeteroNetworkEmbeddingGradient( self.encrypt_operator) fore_gradient = self.gradient_operator.compute_fore_gradient( batch_data_inst, en_aggregate_ee) host_gradient = self.gradient_operator.compute_gradient( self.guest_forward.mapValues( lambda v: Instance(features=v[1])), fore_gradient) federation.remote( host_gradient, name=self.transfer_variable.host_gradient.name, tag=self.transfer_variable.generate_transferid( self.transfer_variable.host_gradient, self.n_iter_, batch_index), role=consts.ARBITER, idx=0) LOGGER.info("Remote host_gradient to arbiter") composed_data_inst = host_forward.join( batch_data_inst, lambda hf, d: Instance(features=hf[1], label=d.label)) guest_gradient, loss = self.gradient_operator.compute_gradient_and_loss( composed_data_inst, fore_gradient, en_aggregate_ee, en_aggregate_ee_square) federation.remote( guest_gradient, name=self.transfer_variable.guest_gradient.name, tag=self.transfer_variable.generate_transferid( self.transfer_variable.guest_gradient, self.n_iter_, batch_index), role=consts.ARBITER, idx=0) LOGGER.info("Remote guest_gradient to arbiter") optim_guest_gradient = federation.get( name=self.transfer_variable.guest_optim_gradient.name, tag=self.transfer_variable.generate_transferid( self.transfer_variable.guest_optim_gradient, self.n_iter_, batch_index), idx=0) LOGGER.info("Get optim_guest_gradient from arbiter") # update node embedding LOGGER.info("Update node embedding") nodeid_join_gradient = batch_data_inst.join( optim_guest_gradient, lambda instance, gradient: (node2id[instance.features], gradient)) self.update_model(nodeid_join_gradient) # update local model that transform attribute to node embedding training_info = { 'iteration': self.n_iter_, 'batch_index': batch_index } self.update_local_model(fore_gradient, batch_data_inst, self.embedding_, **training_info) # loss need to be encrypted !!!!!! federation.remote( loss, name=self.transfer_variable.loss.name, tag=self.transfer_variable.generate_transferid( self.transfer_variable.loss, self.n_iter_, batch_index), role=consts.ARBITER, idx=0) LOGGER.info("Remote loss to arbiter") # is converge of loss in arbiter batch_index += 1 # remove temporary resource rubbish_list = [ host_forward, aggregate_forward_res, en_aggregate_ee, en_aggregate_ee_square, fore_gradient, self.guest_forward ] rubbish_clear(rubbish_list) ########## guest_common_embedding = common_node_instances.mapValues( lambda node: self.embedding_[node2id[node]]) federation.remote( guest_common_embedding, name=self.transfer_variable.guest_common_embedding.name, tag=self.transfer_variable.generate_transferid( self.transfer_variable.guest_common_embedding, self.n_iter_, 1), role=consts.ARBITER, idx=0) common_embedding = federation.get( name=self.transfer_variable.common_embedding.name, tag=self.transfer_variable.generate_transferid( self.transfer_variable.common_embedding, self.n_iter_, 1), idx=0) self.update_common_nodes(common_embedding, common_nodes, node2id) ########## is_stopped = federation.get( name=self.transfer_variable.is_stopped.name, tag=self.transfer_variable.generate_transferid( self.transfer_variable.is_stopped, self.n_iter_), idx=0) LOGGER.info("Get is_stop flag from arbiter:{}".format(is_stopped)) self.n_iter_ += 1 if is_stopped: LOGGER.info( "Get stop signal from arbiter, model is converged, iter:{}" .format(self.n_iter_)) break embedding_table = eggroll.table(name='guest', namespace='node_embedding', partition=10) id2node = dict(zip(node2id.values(), node2id.keys())) for id, embedding in enumerate(self.embedding_): embedding_table.put(id2node[id], embedding) embedding_table.save_as(name='guest', namespace='node_embedding', partition=10) LOGGER.info("Reach max iter {}, train model finish!".format( self.max_iter))
def fit(self, data_instances): """ Train lr model of role guest Parameters ---------- data_instances: DTable of Instance, input data """ LOGGER.info("Enter hetero_lr_guest fit") self._abnormal_detection(data_instances) self.header = self.get_header(data_instances) data_instances = data_instances.mapValues(HeteroLRGuest.load_data) # 获得密钥 public_key = federation.get( name=self.transfer_variable.paillier_pubkey.name, tag=self.transfer_variable.generate_transferid( self.transfer_variable.paillier_pubkey), idx=0) LOGGER.info("Get public_key from arbiter:{}".format(public_key)) self.encrypt_operator.set_public_key(public_key) LOGGER.info("Generate mini-batch from input data") mini_batch_obj = MiniBatch(data_instances, batch_size=self.batch_size) batch_num = mini_batch_obj.batch_nums if self.batch_size == -1: LOGGER.info( "batch size is -1, set it to the number of data in data_instances" ) self.batch_size = data_instances.count() batch_info = {"batch_size": self.batch_size, "batch_num": batch_num} LOGGER.info("batch_info:{}".format(batch_info)) federation.remote(batch_info, name=self.transfer_variable.batch_info.name, tag=self.transfer_variable.generate_transferid( self.transfer_variable.batch_info), role=consts.HOST, idx=0) LOGGER.info("Remote batch_info to Host") federation.remote(batch_info, name=self.transfer_variable.batch_info.name, tag=self.transfer_variable.generate_transferid( self.transfer_variable.batch_info), role=consts.ARBITER, idx=0) LOGGER.info("Remote batch_info to Arbiter") self.encrypted_calculator = [ EncryptModeCalculator( self.encrypt_operator, self.encrypted_mode_calculator_param.mode, self.encrypted_mode_calculator_param.re_encrypted_rate) for _ in range(batch_num) ] LOGGER.info("Start initialize model.") LOGGER.info("fit_intercept:{}".format( self.init_param_obj.fit_intercept)) model_shape = self.get_features_shape(data_instances) weight = self.initializer.init_model(model_shape, init_params=self.init_param_obj) if self.init_param_obj.fit_intercept is True: self.coef_ = weight[:-1] self.intercept_ = weight[-1] else: self.coef_ = weight is_send_all_batch_index = False self.n_iter_ = 0 index_data_inst_map = {} while self.n_iter_ < self.max_iter: LOGGER.info("iter:{}".format(self.n_iter_)) # each iter will get the same batch_data_generator batch_data_generator = mini_batch_obj.mini_batch_data_generator( result='index') batch_index = 0 for batch_data_index in batch_data_generator: LOGGER.info("batch:{}".format(batch_index)) if not is_send_all_batch_index: LOGGER.info("remote mini-batch index to Host") federation.remote( batch_data_index, name=self.transfer_variable.batch_data_index.name, tag=self.transfer_variable.generate_transferid( self.transfer_variable.batch_data_index, self.n_iter_, batch_index), role=consts.HOST, idx=0) if batch_index >= mini_batch_obj.batch_nums - 1: is_send_all_batch_index = True # Get mini-batch train data if len(index_data_inst_map) < batch_num: batch_data_inst = data_instances.join( batch_data_index, lambda data_inst, index: data_inst) index_data_inst_map[batch_index] = batch_data_inst else: batch_data_inst = index_data_inst_map[batch_index] # transforms features of raw input 'batch_data_inst' into more representative features 'batch_feat_inst' batch_feat_inst = self.transform(batch_data_inst) # guest/host forward self.compute_forward(batch_feat_inst, self.coef_, self.intercept_, batch_index) host_forward = federation.get( name=self.transfer_variable.host_forward_dict.name, tag=self.transfer_variable.generate_transferid( self.transfer_variable.host_forward_dict, self.n_iter_, batch_index), idx=0) LOGGER.info("Get host_forward from host") aggregate_forward_res = self.aggregate_forward(host_forward) en_aggregate_wx = aggregate_forward_res.mapValues( lambda v: v[0]) en_aggregate_wx_square = aggregate_forward_res.mapValues( lambda v: v[1]) # compute [[d]] if self.gradient_operator is None: self.gradient_operator = HeteroLogisticGradient( self.encrypt_operator) fore_gradient = self.gradient_operator.compute_fore_gradient( batch_feat_inst, en_aggregate_wx) federation.remote( fore_gradient, name=self.transfer_variable.fore_gradient.name, tag=self.transfer_variable.generate_transferid( self.transfer_variable.fore_gradient, self.n_iter_, batch_index), role=consts.HOST, idx=0) LOGGER.info("Remote fore_gradient to Host") # compute guest gradient and loss guest_gradient, loss = self.gradient_operator.compute_gradient_and_loss( batch_feat_inst, fore_gradient, en_aggregate_wx, en_aggregate_wx_square, self.fit_intercept) # loss regulation if necessary if self.updater is not None: guest_loss_regular = self.updater.loss_norm(self.coef_) loss += self.encrypt_operator.encrypt(guest_loss_regular) federation.remote( guest_gradient, name=self.transfer_variable.guest_gradient.name, tag=self.transfer_variable.generate_transferid( self.transfer_variable.guest_gradient, self.n_iter_, batch_index), role=consts.ARBITER, idx=0) LOGGER.info("Remote guest_gradient to arbiter") optim_guest_gradient = federation.get( name=self.transfer_variable.guest_optim_gradient.name, tag=self.transfer_variable.generate_transferid( self.transfer_variable.guest_optim_gradient, self.n_iter_, batch_index), idx=0) LOGGER.info("Get optim_guest_gradient from arbiter") # update model LOGGER.info("update_model") self.update_model(optim_guest_gradient) # update local model that transforms features of raw input 'batch_data_inst' training_info = { "iteration": self.n_iter_, "batch_index": batch_index } self.update_local_model(fore_gradient, batch_data_inst, self.coef_, **training_info) # Get loss regulation from Host if regulation is set if self.updater is not None: en_host_loss_regular = federation.get( name=self.transfer_variable.host_loss_regular.name, tag=self.transfer_variable.generate_transferid( self.transfer_variable.host_loss_regular, self.n_iter_, batch_index), idx=0) LOGGER.info("Get host_loss_regular from Host") loss += en_host_loss_regular federation.remote( loss, name=self.transfer_variable.loss.name, tag=self.transfer_variable.generate_transferid( self.transfer_variable.loss, self.n_iter_, batch_index), role=consts.ARBITER, idx=0) LOGGER.info("Remote loss to arbiter") # is converge of loss in arbiter batch_index += 1 # temporary resource recovery and will be removed in the future rubbish_list = [ host_forward, aggregate_forward_res, en_aggregate_wx, en_aggregate_wx_square, fore_gradient, self.guest_forward ] rubbish_clear(rubbish_list) is_stopped = federation.get( name=self.transfer_variable.is_stopped.name, tag=self.transfer_variable.generate_transferid( self.transfer_variable.is_stopped, self.n_iter_, batch_index), idx=0) LOGGER.info("Get is_stop flag from arbiter:{}".format(is_stopped)) self.n_iter_ += 1 if is_stopped: LOGGER.info( "Get stop signal from arbiter, model is converged, iter:{}" .format(self.n_iter_)) break LOGGER.info("Reach max iter {}, train model finish!".format( self.max_iter))
def fit(self, data_instances): """ Train lr model of role host Parameters ---------- data_instances: DTable of Instance, input data """ LOGGER.info("Enter hetero_lr host") self._abnormal_detection(data_instances) self.header = self.get_header(data_instances) public_key = federation.get( name=self.transfer_variable.paillier_pubkey.name, tag=self.transfer_variable.generate_transferid( self.transfer_variable.paillier_pubkey), idx=0) LOGGER.info("Get public_key from arbiter:{}".format(public_key)) self.encrypt_operator.set_public_key(public_key) batch_info = federation.get( name=self.transfer_variable.batch_info.name, tag=self.transfer_variable.generate_transferid( self.transfer_variable.batch_info), idx=0) LOGGER.info("Get batch_info from guest:" + str(batch_info)) self.batch_size = batch_info["batch_size"] self.batch_num = batch_info["batch_num"] if self.batch_size < consts.MIN_BATCH_SIZE and self.batch_size != -1: raise ValueError( "Batch size get from guest should not less than 10, except -1, batch_size is {}" .format(self.batch_size)) self.encrypted_calculator = [ EncryptModeCalculator( self.encrypt_operator, self.encrypted_mode_calculator_param.mode, self.encrypted_mode_calculator_param.re_encrypted_rate) for _ in range(self.batch_num) ] LOGGER.info("Start initialize model.") model_shape = self.get_features_shape(data_instances) if self.init_param_obj.fit_intercept: self.init_param_obj.fit_intercept = False if self.fit_intercept: self.fit_intercept = False self.coef_ = self.initializer.init_model( model_shape, init_params=self.init_param_obj) self.n_iter_ = 0 index_data_inst_map = {} while self.n_iter_ < self.max_iter: LOGGER.info("iter:" + str(self.n_iter_)) batch_index = 0 while batch_index < self.batch_num: LOGGER.info("batch:{}".format(batch_index)) # set batch_data if len(self.batch_index_list) < self.batch_num: batch_data_index = federation.get( name=self.transfer_variable.batch_data_index.name, tag=self.transfer_variable.generate_transferid( self.transfer_variable.batch_data_index, self.n_iter_, batch_index), idx=0) LOGGER.info("Get batch_index from Guest") self.batch_index_list.append(batch_data_index) else: batch_data_index = self.batch_index_list[batch_index] # Get mini-batch train data if len(index_data_inst_map) < self.batch_num: batch_data_inst = batch_data_index.join( data_instances, lambda g, d: d) index_data_inst_map[batch_index] = batch_data_inst else: batch_data_inst = index_data_inst_map[batch_index] LOGGER.info("batch_data_inst size:{}".format( batch_data_inst.count())) # transforms features of raw input 'batch_data_inst' into more representative features 'batch_feat_inst' batch_feat_inst = self.transform(batch_data_inst) # compute forward host_forward = self.compute_forward(batch_feat_inst, self.coef_, self.intercept_, batch_index) federation.remote( host_forward, name=self.transfer_variable.host_forward_dict.name, tag=self.transfer_variable.generate_transferid( self.transfer_variable.host_forward_dict, self.n_iter_, batch_index), role=consts.GUEST, idx=0) LOGGER.info("Remote host_forward to guest") # compute host gradient fore_gradient = federation.get( name=self.transfer_variable.fore_gradient.name, tag=self.transfer_variable.generate_transferid( self.transfer_variable.fore_gradient, self.n_iter_, batch_index), idx=0) LOGGER.info("Get fore_gradient from guest") if self.gradient_operator is None: self.gradient_operator = HeteroLogisticGradient( self.encrypt_operator) host_gradient = self.gradient_operator.compute_gradient( batch_feat_inst, fore_gradient, fit_intercept=False) # regulation if necessary if self.updater is not None: loss_regular = self.updater.loss_norm(self.coef_) en_loss_regular = self.encrypt_operator.encrypt( loss_regular) federation.remote( en_loss_regular, name=self.transfer_variable.host_loss_regular.name, tag=self.transfer_variable.generate_transferid( self.transfer_variable.host_loss_regular, self.n_iter_, batch_index), role=consts.GUEST, idx=0) LOGGER.info("Remote host_loss_regular to guest") federation.remote( host_gradient, name=self.transfer_variable.host_gradient.name, tag=self.transfer_variable.generate_transferid( self.transfer_variable.host_gradient, self.n_iter_, batch_index), role=consts.ARBITER, idx=0) LOGGER.info("Remote host_gradient to arbiter") # Get optimize host gradient and update model optim_host_gradient = federation.get( name=self.transfer_variable.host_optim_gradient.name, tag=self.transfer_variable.generate_transferid( self.transfer_variable.host_optim_gradient, self.n_iter_, batch_index), idx=0) LOGGER.info("Get optim_host_gradient from arbiter") LOGGER.info("update_model") self.update_model(optim_host_gradient) # update local model that transforms features of raw input 'batch_data_inst' training_info = { "iteration": self.n_iter_, "batch_index": batch_index } self.update_local_model(fore_gradient, batch_data_inst, self.coef_, **training_info) batch_index += 1 # temporary resource recovery and will be removed in the future rubbish_list = [host_forward, fore_gradient] data_overview.rubbish_clear(rubbish_list) is_stopped = federation.get( name=self.transfer_variable.is_stopped.name, tag=self.transfer_variable.generate_transferid( self.transfer_variable.is_stopped, self.n_iter_, batch_index), idx=0) LOGGER.info("Get is_stop flag from arbiter:{}".format(is_stopped)) self.n_iter_ += 1 if is_stopped: LOGGER.info( "Get stop signal from arbiter, model is converged, iter:{}" .format(self.n_iter_)) break LOGGER.info("Reach max iter {}, train model finish!".format( self.max_iter))
def fit(self, data_instances, node2id): """ Train ne model pf role host Parameters ---------- data_instances: Dtable of anchor node, input data """ LOGGER.info("Enter hetero_ne host") self.n_node = len(node2id) LOGGER.info("Host party has {} nodes".format(self.n_node)) data_instances = data_instances.mapValues(HeteroNEHost.load_data) LOGGER.info("Transform input data to train instance") public_key = federation.get( name=self.transfer_variable.paillier_pubkey.name, tag=self.transfer_variable.generate_transferid( self.transfer_variable.paillier_pubkey), idx=0) LOGGER.info("Get Publick key from arbiter:{}".format(public_key)) self.encrypt_operator.set_public_key(public_key) batch_info = federation.get( name=self.transfer_variable.batch_info.name, tag=self.transfer_variable.generate_transferid( self.transfer_variable.batch_info), idx=0) LOGGER.info("Get batch_info from guest: {}".format(batch_info)) self.batch_size = batch_info['batch_size'] self.batch_num = batch_info['batch_num'] if self.batch_size < consts.MIN_BATCH_SIZE and self.batch_size != -1: raise ValueError( "Batch size get from guest should not less than 10, except -1, batch_size is {}" .format(self.batch_size)) self.encrypted_calculator = [ EncryptModeCalculator( self.encrypt_operator, self.encrypted_mode_calculator_param.mode, self.encrypted_mode_calculator_param.re_encrypted_rate) for _ in range(self.batch_num) ] LOGGER.info("Start initilize model.") self.embedding_ = self.initializer.init_model((self.n_node, self.dim), self.init_param_obj) self.n_iter_ = 0 index_data_inst_map = {} while self.n_iter_ < self.max_iter: LOGGER.info("iter: {}".format(self.n_iter_)) batch_index = 0 while batch_index < self.batch_num: LOGGER.info("batch:{}".format(batch_index)) # set batch_data # in order to avoid communicating in next iteration # in next iteration, the sequence of batches is the same if len(self.batch_index_list) < self.batch_num: batch_data_index = federation.get( name=self.transfer_variable.batch_data_index.name, tag=self.transfer_variable.generate_transferid( self.transfer_variable.batch_data_index, self.n_iter_, batch_index), idx=0) LOGGER.info("Get batch_index from Guest") self.batch_index_list.append(batch_index) else: batch_data_index = self.batch_index_list[batch_index] # Get mini-batch train_data # in order to avoid joining for next iteration if len(index_data_inst_map) < self.batch_num: batch_data_inst = batch_data_index.join( data_instances, lambda g, d: d) index_data_inst_map[batch_index] = batch_data_inst else: batch_data_inst = index_data_inst_map[batch_data_index] LOGGER.info("batch_data_inst size:{}".format( batch_data_inst.count())) #self.transform(data_inst) # compute forward host_forward = self.compute_forward(batch_data_inst, self.embedding_, node2id, batch_index) federation.remote( host_forward, name=self.transfer_variable.host_forward_dict.name, tag=self.transfer_variable.generate_transferid( self.transfer_variable.host_forward_dict, self.n_iter_, batch_index), role=consts.GUEST, idx=0) LOGGER.info("Remote host_forward to guest") # Get optimize host gradient and update model optim_host_gradient = federation.get( name=self.transfer_variable.host_optim_gradient.name, tag=self.transfer_variable.generate_transferid( self.transfer_variable.host_optim_gradient, self.n_iter_, batch_index), idx=0) LOGGER.info("Get optim_host_gradient from arbiter") nodeid_join_gradient = batch_data_inst.join( optim_host_gradient, lambda instance, gradient: (node2id[instance.features], gradient)) LOGGER.info("update_model") self.update_model(nodeid_join_gradient) # update local model #training_info = {"iteration": self.n_iter_, "batch_index": batch_index} #self.update_local_model(fore_gradient, batch_data_inst, self.coef_, **training_info) batch_index += 1 rubbish_list = [host_forward] rubbish_clear(rubbish_list) is_stopped = federation.get( name=self.transfer_variable.is_stopped.name, tag=self.transfer_variable.generate_transferid( self.transfer_variable.is_stopped, self.n_iter_, )) LOGGER.info("Get is_stop flag from arbiter:{}".format(is_stopped)) self.n_iter_ += 1 if is_stopped: LOGGER.info("Reach max iter {}, train mode finish!".format( self.max_iter)) embedding_table = eggroll.table(name='host', namespace='node_embedding', partition=10) id2node = dict(zip(node2id.values(), node2id.keys())) for id, embedding in enumerate(self.embedding_): embedding_table.put(id2node[id], embedding) embedding_table.save_as(name='host', namespace='node_embedding', partition=10) LOGGER.info("Reach max iter {}, train model finish!".format( self.max_iter))
def fit(self, data_instances, node2id, local_instances=None, common_nodes=None): """ Train ne model pf role host Parameters ---------- data_instances: Dtable of anchor node, input data """ LOGGER.info("Enter hetero_ne host") self.n_node = len(node2id) LOGGER.info("Host party has {} nodes".format(self.n_node)) data_instances = data_instances.mapValues(HeteroNEHost.load_data) LOGGER.info("Transform input data to train instance") public_key = federation.get( name=self.transfer_variable.paillier_pubkey.name, tag=self.transfer_variable.generate_transferid( self.transfer_variable.paillier_pubkey), idx=0) LOGGER.info("Get Publick key from arbiter:{}".format(public_key)) self.encrypt_operator.set_public_key(public_key) ############## # horizontal federated learning LOGGER.info("Generate mini-batch for local instances in guest") mini_batch_obj_local = MiniBatch(local_instances, batch_size=self.batch_size) common_node_instances = eggroll.parallelize( ((node, node) for node in common_nodes), include_key=True, name='common_nodes') ############## batch_info = federation.get( name=self.transfer_variable.batch_info.name, tag=self.transfer_variable.generate_transferid( self.transfer_variable.batch_info), idx=0) LOGGER.info("Get batch_info from guest: {}".format(batch_info)) self.batch_size = batch_info['batch_size'] self.batch_num = batch_info['batch_num'] if self.batch_size < consts.MIN_BATCH_SIZE and self.batch_size != -1: raise ValueError( "Batch size get from guest should not less than 10, except -1, batch_size is {}" .format(self.batch_size)) self.encrypted_calculator = [ EncryptModeCalculator( self.encrypt_operator, self.encrypted_mode_calculator_param.mode, self.encrypted_mode_calculator_param.re_encrypted_rate) for _ in range(self.batch_num) ] LOGGER.info("Start initilize model.") self.embedding_ = self.initializer.init_model((self.n_node, self.dim), self.init_param_obj) self.n_iter_ = 0 index_data_inst_map = {} while self.n_iter_ < self.max_iter: LOGGER.info("iter: {}".format(self.n_iter_)) ################# local_batch_data_generator = mini_batch_obj_local.mini_batch_data_generator( ) total_loss = 0 local_batch_num = 0 LOGGER.info("Horizontally learning") for local_batch_data in local_batch_data_generator: n = local_batch_data.count() LOGGER.info("Local batch data count:{}".format(n)) E_Y = self.compute_local_embedding(local_batch_data, self.embedding_, node2id) local_grads_e1, local_grads_e2, local_loss = self.local_gradient_operator.compute( E_Y, 'E_1') local_grads_e1 = local_grads_e1.mapValues( lambda g: self.local_optimizer.apply_gradients(g / n)) local_grads_e2 = local_grads_e2.mapValues( lambda g: self.local_optimizer.apply_gradients(g / n)) e1id_join_grads = local_batch_data.join( local_grads_e1, lambda v, g: (node2id[v[0]], g)) e2id_join_grads = local_batch_data.join( local_grads_e2, lambda v, g: (node2id[v[1]], g)) self.update_model(e1id_join_grads) self.update_model(e2id_join_grads) local_loss = local_loss / n local_batch_num += 1 total_loss += local_loss LOGGER.info("gradient count:{}".format( e1id_join_grads.count())) host_common_embedding = common_node_instances.mapValues( lambda node: self.embedding_[node2id[node]]) federation.remote( host_common_embedding, name=self.transfer_variable.host_common_embedding.name, tag=self.transfer_variable.generate_transferid( self.transfer_variable.host_common_embedding, self.n_iter_, 0), role=consts.ARBITER, idx=0) common_embedding = federation.get( name=self.transfer_variable.common_embedding.name, tag=self.transfer_variable.generate_transferid( self.transfer_variable.common_embedding, self.n_iter_, 0), idx=0) self.update_common_nodes(common_embedding, common_nodes, node2id) total_loss /= local_batch_num LOGGER.info("Iter {}, Local loss: {}".format( self.n_iter_, total_loss)) batch_index = 0 while batch_index < self.batch_num: LOGGER.info("batch:{}".format(batch_index)) # set batch_data # in order to avoid communicating in next iteration # in next iteration, the sequence of batches is the same if len(self.batch_index_list) < self.batch_num: batch_data_index = federation.get( name=self.transfer_variable.batch_data_index.name, tag=self.transfer_variable.generate_transferid( self.transfer_variable.batch_data_index, self.n_iter_, batch_index), idx=0) LOGGER.info("Get batch_index from Guest") self.batch_index_list.append(batch_index) else: batch_data_index = self.batch_index_list[batch_index] # Get mini-batch train_data # in order to avoid joining for next iteration if len(index_data_inst_map) < self.batch_num: batch_data_inst = batch_data_index.join( data_instances, lambda g, d: d) index_data_inst_map[batch_index] = batch_data_inst else: batch_data_inst = index_data_inst_map[batch_data_index] LOGGER.info("batch_data_inst size:{}".format( batch_data_inst.count())) #self.transform(data_inst) # compute forward host_forward = self.compute_forward(batch_data_inst, self.embedding_, node2id, batch_index) federation.remote( host_forward, name=self.transfer_variable.host_forward_dict.name, tag=self.transfer_variable.generate_transferid( self.transfer_variable.host_forward_dict, self.n_iter_, batch_index), role=consts.GUEST, idx=0) LOGGER.info("Remote host_forward to guest") # Get optimize host gradient and update model optim_host_gradient = federation.get( name=self.transfer_variable.host_optim_gradient.name, tag=self.transfer_variable.generate_transferid( self.transfer_variable.host_optim_gradient, self.n_iter_, batch_index), idx=0) LOGGER.info("Get optim_host_gradient from arbiter") nodeid_join_gradient = batch_data_inst.join( optim_host_gradient, lambda instance, gradient: (node2id[instance.features], gradient)) LOGGER.info("update_model") self.update_model(nodeid_join_gradient) # update local model #training_info = {"iteration": self.n_iter_, "batch_index": batch_index} #self.update_local_model(fore_gradient, batch_data_inst, self.coef_, **training_info) batch_index += 1 rubbish_list = [host_forward] rubbish_clear(rubbish_list) ####### host_common_embedding = common_node_instances.mapValues( lambda node: self.embedding_[node2id[node]]) federation.remote( host_common_embedding, name=self.transfer_variable.host_common_embedding.name, tag=self.transfer_variable.generate_transferid( self.transfer_variable.host_common_embedding, self.n_iter_, 1), role=consts.ARBITER, idx=0) common_embedding = federation.get( name=self.transfer_variable.common_embedding.name, tag=self.transfer_variable.generate_transferid( self.transfer_variable.common_embedding, self.n_iter_, 1), idx=0) self.update_common_nodes(common_embedding, common_nodes, node2id) ####### is_stopped = federation.get( name=self.transfer_variable.is_stopped.name, tag=self.transfer_variable.generate_transferid( self.transfer_variable.is_stopped, self.n_iter_, ), idx=0) LOGGER.info("Get is_stop flag from arbiter:{}".format(is_stopped)) self.n_iter_ += 1 if is_stopped: break LOGGER.info("Reach max iter {}, train mode finish!".format( self.max_iter)) embedding_table = eggroll.table(name='host', namespace='node_embedding', partition=10) id2node = dict(zip(node2id.values(), node2id.keys())) for id, embedding in enumerate(self.embedding_): embedding_table.put(id2node[id], embedding) embedding_table.save_as(name='host', namespace='node_embedding', partition=10) LOGGER.info("Reach max iter {}, train model finish!".format( self.max_iter))