def forward(self, **kwargs): data: Dataset = kwargs['data'] if 'seperator' not in kwargs: seperator = FeaSeperator(data.name).get_seperator() else: seperator: FeaSeperator = kwargs['seperator'] fea_seperator = seperator.get_seperator() n_rules_tree = seperator.get_n_rule_tree() data_tmp = data neuron_seed = self.get_neuron_seed() rules_tree: List[List[type(neuron_seed)]] = [] for j in torch.arange(len(fea_seperator)): sub_seperator = fea_seperator[int(j)] if len(sub_seperator): sub_dataset_list = data_tmp.get_subset_fea(sub_seperator) # set level j rules_sub: List[type(neuron_seed)] = [] y_sub = torch.empty(data.Y.shape[0], 0).double() for i in torch.arange(len(sub_seperator)): neuron_seed.clear() neuron_c = neuron_seed.clone() sub_dataset_i = sub_dataset_list[i] kwargs['data'] = sub_dataset_i kwargs['n_rules'] = int(n_rules_tree[j][i]) neuron_c.forward(**kwargs) y_i = neuron_c.predict(sub_dataset_i) rules_sub.append(neuron_c) y_sub = torch.cat((y_sub, y_i), 1) # set outputs of this level as dataset data_tmp = Dataset(f"{data}_level{int(j + 1)}", y_sub, data.Y, data.task) rules_tree.append(rules_sub) else: # set bottom level kwargs['data'] = data_tmp kwargs['n_rules'] = int(n_rules_tree[j][0]) neuron_seed.clear() neuron = neuron_seed.clone() neuron.forward(**kwargs) # set bottom level neuron rules_btm: List[type(neuron_seed)] = [neuron] rules_tree.append(rules_btm) self.set_neuron_tree(rules_tree)
def forward(self, **kwargs): data: Dataset = kwargs['data'] if 'seperator' not in kwargs: seperator = FeaSeperator(data.name).get_seperator() else: seperator: FeaSeperator = kwargs['seperator'] fea_seperator = seperator.get_seperator() n_rules_tree = seperator.get_n_rule_tree() data_tmp = data neuron_seed = self.get_neuron_seed() output = None w_sub = [] rules_tree: List[List[type(neuron_seed)]] = [] for j in torch.arange(len(fea_seperator)): sub_seperator = fea_seperator[int(j)] if len(sub_seperator): sub_dataset_list = data_tmp.get_subset_fea(sub_seperator) # get the input of below net work rules_sub: List[type(neuron_seed)] = [] for i in torch.arange(len(sub_seperator)): neuron_seed.clear() neuron_c = neuron_seed.clone() sub_dataset_i = sub_dataset_list[i] kwargs['data'] = sub_dataset_i kwargs['n_rules'] = int(n_rules_tree[j][i]) neuron_c.forward(**kwargs) w_sub.append(neuron_c.get_rules().consequent_list) rules_sub.append(neuron_c) rules_tree.append(rules_sub) else: # creat neural net work loss_fn = nn.CrossEntropyLoss() n_neuron = len(fea_seperator[0]) n_rules = int(n_rules_tree[0][0]) model_foot: ConsequentNet = ConsequentNet(n_neuron, n_rules) optimizer = torch.optim.Adam(model_foot.parameters(), lr=0.0001) epochs = 1500 train_losses = [] for epoch in range(epochs): model_foot.train() outputs = model_foot(w_sub) loss = loss_fn(outputs.double(), data.Y.long().squeeze(1)) loss.backward() optimizer.step() train_losses.append(loss.item()) print( f"epoch : {epoch + 1}, train loss : {train_losses[-1]} " ) # validate the model model_foot.eval() with torch.no_grad(): outputs = model_foot(w_sub) loss = loss_fn(outputs, data.Y.long().squeeze(1)) valid_losses = loss.item() _, predicted = torch.max(outputs.data, 1) correct = ( predicted == data.Y.squeeze().long()).sum().item() total = data.Y.size(0) accuracy = 100 * correct / total print( f"valid loss : {valid_losses}, valid acc : {accuracy}%" ) self.set_lei_net(model_foot) self.set_neuron_tree(rules_tree) return output
def forward(self, **kwargs): data: Dataset = kwargs['data'] if 'seperator' not in kwargs: seperator = FeaSeperator(data.name).get_seperator() else: seperator: FeaSeperator = kwargs['seperator'] fea_seperator = seperator.get_seperator() n_rules_tree = seperator.get_n_rule_tree() neuron_seed = self.get_neuron_seed() sub_seperator = fea_seperator[int(0)] sub_dataset_list = data.get_subset_fea(sub_seperator) sub_dataset_tmp = sub_dataset_list[0] n_rule_tmp = int(n_rules_tree[0][0]) n_smpl = sub_dataset_tmp.X.shape[0] n_fea_tmp = sub_dataset_tmp.X.shape[1] + 1 n_h = n_rule_tmp * n_fea_tmp h_all = torch.empty(0, n_smpl, n_h).double() n_branch = len(sub_seperator) # get neuron tree rules_tree: List[List[type(neuron_seed)]] = [] rules_sub: List[type(neuron_seed)] = [] # get output of every branches of upper dfnn layer for i in torch.arange(n_branch): neuron_seed.clear() neuron_c = neuron_seed.clone() sub_dataset_i = sub_dataset_list[i] kwargs['data'] = sub_dataset_i kwargs['n_rules'] = int(n_rules_tree[0][i]) neuron_c.forward(**kwargs) rules_sub.append(neuron_c) # get rules in neuron and update centers and bias rule_ao = neuron_c.get_rules() # get h computer in neuron h_computer_ao = neuron_c.get_h_computer() h_tmp, _ = h_computer_ao.comute_h(sub_dataset_i.X, rule_ao) h_cal_tmp = h_tmp.permute((1, 0, 2)) # N * n_rules * (d + 1) h_cal_tmp: torch.Tensor = h_cal_tmp.reshape(n_smpl, n_h) h_all = torch.cat((h_all, h_cal_tmp.unsqueeze(0)), 0) rules_tree.append(rules_sub) self.set_neuron_tree(rules_tree) # set consequent net consequent_net: ConsequentNet = ConsequentNet(n_branch, n_h) optimizer = torch.optim.Adam(consequent_net.parameters(), lr=0.001) loss_fn = nn.CrossEntropyLoss() # transform h tensor to torch dataset train_dataset = DatasetH(x=h_all, y=data.Y) train_loader = DataLoader(dataset=train_dataset, batch_size=n_smpl, shuffle=False) epochs = 3000 train_th = 0.000001 train_losses = [] for epoch in range(epochs): consequent_net.train() for i, (h_batch, labels) in enumerate(train_loader): optimizer.zero_grad() outputs = consequent_net(h_batch) loss = loss_fn(outputs.double(), labels.long().squeeze(1)) loss.backward() optimizer.step() train_losses.append(loss.item()) print(f"epoch : {epoch + 1}, train loss : {train_losses[-1]} ") # if len(train_losses) >= 2: # loss_diff = abs(train_losses[-1] - train_losses[len(train_losses)-2]) # # when the whole model stops to update # if loss_diff < train_th: # break # validate the model consequent_net.eval() correct = 0 total = 0 with torch.no_grad(): for i, (h_batch, labels) in enumerate(train_loader): outputs = consequent_net(h_batch) # loss = loss_fn(outputs, labels.long().squeeze(1)) # valid_losses = loss.item() _, predicted = torch.max(outputs.data, 1) correct += ( predicted == labels.squeeze().long()).sum().item() total += labels.size(0) # print(f"valid loss : {valid_losses}") accuracy = 100 * correct / total print(f"train acc : {accuracy}%") self.__consequent_net = consequent_net
def forward(self, **kwargs): train_data: Dataset = kwargs['train_data'] test_data: Dataset = kwargs['test_data'] if 'seperator' not in kwargs: seperator = FeaSeperator(train_data.name).get_seperator() else: seperator: FeaSeperator = kwargs['seperator'] para_mu = kwargs['para_mu'] para_mu1 = kwargs['para_mu1'] fea_seperator = seperator.get_seperator() seperator.generate_n_rule_tree(kwargs['n_rules']) n_rules_tree = seperator.get_n_rule_tree() neuron_seed = self.get_neuron_seed() sub_seperator = fea_seperator[int(0)] sub_dataset_list = train_data.get_subset_fea(sub_seperator) sub_dataset_tmp = sub_dataset_list[0] n_rule_tmp = int(n_rules_tree[0][0]) n_smpl = sub_dataset_tmp.X.shape[0] n_fea_tmp = sub_dataset_tmp.X.shape[1] + 1 n_h = n_rule_tmp * n_fea_tmp h_all = torch.empty(0, n_smpl, n_h).double() n_branch = len(sub_seperator) # get neuron tree rules_tree: List[List[type(neuron_seed)]] = [] rules_sub: List[type(neuron_seed)] = [] # get output of every branches of upper dfnn layer for i in torch.arange(n_branch): neuron_seed.clear() neuron_c = neuron_seed.clone() sub_dataset_i = sub_dataset_list[i] kwargs['data'] = sub_dataset_i kwargs['n_rules'] = int(n_rules_tree[0][i]) neuron_c.forward(**kwargs) rules_sub.append(neuron_c) # get rules in neuron and update centers and bias rule_ao = neuron_c.get_rules() # get h computer in neuron h_computer_ao = neuron_c.get_h_computer() h_tmp, _ = h_computer_ao.comute_h(sub_dataset_i.X, rule_ao) h_cal_tmp = h_tmp.permute((1, 0, 2)) # N * n_rules * (d + 1) h_cal_tmp: torch.Tensor = h_cal_tmp.reshape(n_smpl, n_h) h_all = torch.cat((h_all, h_cal_tmp.unsqueeze(0)), 0) rules_tree.append(rules_sub) self.set_neuron_tree(rules_tree) # set bottom level AO n_midle_output = kwargs['n_hidden_output'] w_x = torch.rand(n_branch, n_midle_output, n_h).double() w_y = torch.rand(n_branch * n_midle_output, 1).double() # start AO optimization diff = 1 loss = 100 train_loss_list = [] run_th = 0.00001 n_epoch = 20 run_epoch = 1 loss_test_old = 100 # get h_all h_brunch_all = torch.empty(n_smpl, 0).double() for i in torch.arange(n_branch): h_brunch = h_all[i, :, :].repeat(1, n_midle_output) h_brunch_all = torch.cat((h_brunch_all, h_brunch), 1) # load better parameters data_save_dir = f"./para_file/{kwargs['dataset_name']}" if not os.path.exists(data_save_dir): os.makedirs(data_save_dir) para_file = f"{data_save_dir}/ao_{kwargs['n_rules']}_{kwargs['n_hidden_output']}.pt" # if os.path.exists(para_fi."] g_w_x_best = None g_w_y_best = None # while diff > run_th and run_epoch < n_epoch: while run_epoch < n_epoch: # fix w_y update w_x w_y_brunch_all = w_y.repeat(1, n_h) w_y_brunch_all = w_y_brunch_all.reshape(1, -1) w_x_h_brunch = torch.mul(h_brunch_all, w_y_brunch_all) w_x_brunch = torch.inverse( w_x_h_brunch.t().mm(w_x_h_brunch) + para_mu * torch.eye(w_x_h_brunch.shape[1]).double()).mm( w_x_h_brunch.t().mm(train_data.Y)) w_x_cal = w_x_brunch.squeeze().reshape(n_branch, n_midle_output * n_h) w_x = w_x_cal.reshape(n_branch, n_midle_output, n_h) # fix w_x update w_y w_y_h_cal = torch.empty(n_smpl, 0).double() for i in torch.arange(n_branch): w_y_h_brunch = h_all[i, :, :].mm(w_x[i, :, :].t()).squeeze() if n_midle_output == 1: w_y_h_cal = torch.cat( (w_y_h_cal, w_y_h_brunch.unsqueeze(1)), 1) else: w_y_h_cal = torch.cat((w_y_h_cal, w_y_h_brunch), 1) w_y = torch.inverse(w_y_h_cal.t().mm(w_y_h_cal) + para_mu1 * torch.eye(w_y_h_cal.shape[1]).double()).mm( w_y_h_cal.t().mm(train_data.Y)) # compute loss y_tmp = train_data.Y y_hap_tmp = w_y_h_cal.mm(w_y) # loss_tmp = torch.norm(y_tmp - y_hap_tmp) loss_tmp = mean_squared_error(y_tmp, y_hap_tmp) diff = abs(loss_tmp - loss) loss = loss_tmp train_loss_list.append(loss) self.__w_x = w_x self.__w_y = w_y # snoop the test data performance test_y = self.predict(test_data, kwargs['seperator']) loss_train = mean_squared_error(y_tmp, y_hap_tmp) loss_test = mean_squared_error(test_y, test_data.Y) # print(f"Loss of test: {loss_test}") if loss_test < loss_test_old and loss_test >= loss_train: g_w_x_best = w_x g_w_y_best = w_y run_epoch = run_epoch + 1 # print(f"Loss of AO: {loss} w_y: {w_y.squeeze()[1:5]}") # x = torch.linspace(1, len(train_loss_list) + 1, len(train_loss_list)).numpy() # y = train_loss_list # plt.title('Result Analysis') # plt.plot(x, y, color='green', label='loss value') # # plt.plot(x, test_acys, color='red', label='training accuracy') # plt.legend() # 显示图例 # # plt.xlabel('iteration epochs') # plt.ylabel('loss value') # plt.show() self.__w_x = w_x self.__w_y = w_y if g_w_x_best is not None: self.__w_x = g_w_x_best self.__w_y = g_w_y_best para_dict = dict() para_dict["w_x"] = w_x para_dict["w_y"] = w_y torch.save(para_dict, para_file)