def predict(self, data: Dataset, seperator_p: FeaSeperator = None): neuron_tree = self.get_neuron_tree() neuron_seed = self.get_neuron_seed() if seperator_p is None: fea_seperator = FeaSeperator(data.name).get_seperator() else: fea_seperator = seperator_p.get_seperator() data_tmp = data for i in torch.arange(len(neuron_tree)): output_sub = torch.empty(data.Y.shape[0], 0).double() # get level i rules_sub: List[type(neuron_seed)] = neuron_tree[int(i)] sub_dataset_list = data_tmp.get_subset_fea(fea_seperator[int(i)]) for j in torch.arange(len(rules_sub)): neuron = rules_sub[int(j)] sub_dataset_j = sub_dataset_list[j] output_j = neuron.predict(sub_dataset_j) output_sub = torch.cat((output_sub, output_j), 1) # set outputs of this level as dataset data_tmp = Dataset(f"{data}_level{int(i + 1)}", output_sub, data.Y, data.task) # get bottom fc layer w = self.get_fc_w() y_hat = data_tmp.X.mm(w) return y_hat
def predict(self, data: Dataset, seperator_p: FeaSeperator = None): neuron_tree = self.get_neuron_tree() neuron_seed = self.get_neuron_seed() if seperator_p is None: fea_seperator = FeaSeperator(data.name).get_seperator() else: fea_seperator = seperator_p.get_seperator() neuron_tmp: Neuron = neuron_tree[0][0] n_rule_tmp = int(neuron_tmp.get_rules().n_rules) sub_dataset_list = data.get_subset_fea(fea_seperator[0]) sub_dataset_tmp = sub_dataset_list[0] n_smpl = sub_dataset_tmp.X.shape[0] n_fea_tmp = sub_dataset_tmp.X.shape[1] + 1 n_h = n_rule_tmp * n_fea_tmp h_all = torch.empty(0, n_smpl, n_h).double() rules_sub: List[type(neuron_seed)] = neuron_tree[0] n_branch = len(rules_sub) for j in torch.arange(n_branch): neuron = rules_sub[int(j)] sub_dataset_j = sub_dataset_list[j] neuron.predict(sub_dataset_j) # get rules in neuron and update centers and bias rule_ao = neuron.get_rules() # get h computer in neuron h_computer_ao = neuron.get_h_computer() h_tmp, _ = h_computer_ao.comute_h(sub_dataset_j.X, rule_ao) h_cal_tmp = h_tmp.permute((1, 0, 2)) # N * n_rules * (d + 1) h_cal_tmp = h_cal_tmp.reshape(n_smpl, n_h) h_all = torch.cat((h_all, h_cal_tmp.unsqueeze(0)), 0) # get consequent net # transform h tensor to torch dataset test_dataset = DatasetH(x=h_all, y=data.Y) test_loader = DataLoader(dataset=test_dataset, batch_size=n_smpl, shuffle=False) consequent_net = self.__consequent_net consequent_net.eval() y_hat = torch.empty(0) with torch.no_grad(): for i, (h_batch, labels) in enumerate(test_loader): outputs = consequent_net(h_batch) _, y_hat_tmp = torch.max(outputs.data, 1) y_hat = torch.cat((y_hat, y_hat_tmp.unsqueeze(0).float()), 1) return y_hat.t().double()
def predict(self, data: Dataset, seperator_p: FeaSeperator = None): neuron_tree = self.get_neuron_tree() neuron_seed = self.get_neuron_seed() if seperator_p is None: fea_seperator = FeaSeperator(data.name).get_seperator() else: fea_seperator = seperator_p.get_seperator() neuron_tmp: Neuron = neuron_tree[0][0] n_rule_tmp = int(neuron_tmp.get_rules().n_rules) sub_dataset_list = data.get_subset_fea(fea_seperator[0]) sub_dataset_tmp = sub_dataset_list[0] n_smpl = sub_dataset_tmp.X.shape[0] n_fea_tmp = sub_dataset_tmp.X.shape[1] + 1 n_h = n_rule_tmp * n_fea_tmp h_all = torch.empty(0, n_smpl, n_h).double() rules_sub: List[type(neuron_seed)] = neuron_tree[0] n_branch = len(rules_sub) for j in torch.arange(n_branch): neuron = rules_sub[int(j)] sub_dataset_j = sub_dataset_list[j] neuron.predict(sub_dataset_j) # get rules in neuron and update centers and bias rule_ao = neuron.get_rules() # get h computer in neuron h_computer_ao = neuron.get_h_computer() h_tmp, _ = h_computer_ao.comute_h(sub_dataset_j.X, rule_ao) h_cal_tmp = h_tmp.permute((1, 0, 2)) # N * n_rules * (d + 1) h_cal_tmp = h_cal_tmp.reshape(n_smpl, n_h) h_all = torch.cat((h_all, h_cal_tmp.unsqueeze(0)), 0) # get bottom layer w_x = self.__w_x w_y = self.__w_y # use w_x w_y_h_cal = torch.empty(n_smpl, 0).double() for i in torch.arange(n_branch): w_y_h_brunch = h_all[i, :, :].mm(w_x[i, :, :].t()).squeeze() if len(w_y_h_brunch.shape) == 2: w_y_h_cal = torch.cat((w_y_h_cal, w_y_h_brunch), 1) else: w_y_h_cal = torch.cat((w_y_h_cal, w_y_h_brunch.unsqueeze(1)), 1) y_hat = w_y_h_cal.mm(w_y) return y_hat
def forward(self, **kwargs): data: Dataset = kwargs['data'] if 'seperator' not in kwargs: seperator = FeaSeperator(data.name).get_seperator() else: seperator: FeaSeperator = kwargs['seperator'] fea_seperator = seperator.get_seperator() n_rules_tree = seperator.get_n_rule_tree() data_tmp = data neuron_seed = self.get_neuron_seed() rules_tree: List[List[type(neuron_seed)]] = [] for j in torch.arange(len(fea_seperator)): sub_seperator = fea_seperator[int(j)] if len(sub_seperator): sub_dataset_list = data_tmp.get_subset_fea(sub_seperator) # set level j rules_sub: List[type(neuron_seed)] = [] y_sub = torch.empty(data.Y.shape[0], 0).double() for i in torch.arange(len(sub_seperator)): neuron_seed.clear() neuron_c = neuron_seed.clone() sub_dataset_i = sub_dataset_list[i] kwargs['data'] = sub_dataset_i kwargs['n_rules'] = int(n_rules_tree[j][i]) neuron_c.forward(**kwargs) y_i = neuron_c.predict(sub_dataset_i) rules_sub.append(neuron_c) y_sub = torch.cat((y_sub, y_i), 1) # set outputs of this level as dataset data_tmp = Dataset(f"{data}_level{int(j + 1)}", y_sub, data.Y, data.task) rules_tree.append(rules_sub) else: # set bottom level kwargs['data'] = data_tmp kwargs['n_rules'] = int(n_rules_tree[j][0]) neuron_seed.clear() neuron = neuron_seed.clone() neuron.forward(**kwargs) # set bottom level neuron rules_btm: List[type(neuron_seed)] = [neuron] rules_tree.append(rules_btm) self.set_neuron_tree(rules_tree)
def predict(self, data: Dataset, seperator_p: FeaSeperator = None): neuron_tree = self.get_neuron_tree() neuron_seed = self.get_neuron_seed() if seperator_p is None: fea_seperator = FeaSeperator(data.name).get_seperator() else: fea_seperator = seperator_p.get_seperator() data_tmp = data w_sub = [] for i in torch.arange(len(neuron_tree)): # get level i rules_sub: List[type(neuron_seed)] = neuron_tree[int(i)] sub_dataset_list = data_tmp.get_subset_fea(fea_seperator[int(i)]) for j in torch.arange(len(rules_sub)): neuron = rules_sub[int(j)] sub_dataset_j = sub_dataset_list[j] output_j = neuron.predict(sub_dataset_j) w_sub.append(output_j) # get bottom fc layer loss_fn = nn.CrossEntropyLoss() model: ConsequentNet = self.get_lei_net() model.eval() with torch.no_grad(): outputs = model(w_sub) loss = loss_fn(outputs, data.Y.long().squeeze(1)) valid_losses = loss.item() _, predicted = torch.max(outputs.data, 1) correct = (predicted == data.Y.squeeze().long()).sum().item() total = data.Y.size(0) accuracy = 100 * correct / total print(f"test loss : {valid_losses}, test acc : {accuracy}%") return accuracy
def get_cur_dataset(self, dataset_idx): dataset_name = self.dataset_list[dataset_idx] config_content = self.config_content fea_seperator = FeaSeperator(dataset_name) # set feature seperator seperator_type = config_content['feature_seperator'] if seperator_type == 'slice_window': window_size = config_content['window_size'] step = config_content['step'] n_level = config_content['n_level'] fea_seperator.set_seperator_by_slice_window( window_size, step, n_level) elif seperator_type == 'stride_window': stride_len = config_content['stride_len'] n_level = config_content['n_level'] fea_seperator.set_seperator_by_stride_window(stride_len, n_level) elif seperator_type == 'random_pick': window_size = config_content['window_size'] n_repeat = config_content['n_repeat_select'] n_level = config_content['n_level'] fea_seperator.set_seperator_by_random_pick(window_size, n_repeat, n_level) elif seperator_type == 'no_seperate': fea_seperator.set_seperator_by_no_seperate() # generate tree of rule numbers according to the seperator fea_seperator.generate_n_rule_tree(self.n_rules) # set rule number tree_rule_spesify = config_content['tree_rule_spesify'] if tree_rule_spesify == 'true': n_rule_pos = config_content['n_rule_pos'] n_rule_spesify = config_content['n_rule_spesify'] fea_seperator.set_n_rule_tree(n_rule_pos[0], n_rule_pos[1], n_rule_spesify) self.fea_seperator = fea_seperator return dataset_name
def forward(self, **kwargs): data: Dataset = kwargs['data'] if 'seperator' not in kwargs: seperator = FeaSeperator(data.name).get_seperator() else: seperator: FeaSeperator = kwargs['seperator'] fea_seperator = seperator.get_seperator() n_rules_tree = seperator.get_n_rule_tree() data_tmp = data neuron_seed = self.get_neuron_seed() output = None w_sub = [] rules_tree: List[List[type(neuron_seed)]] = [] for j in torch.arange(len(fea_seperator)): sub_seperator = fea_seperator[int(j)] if len(sub_seperator): sub_dataset_list = data_tmp.get_subset_fea(sub_seperator) # get the input of below net work rules_sub: List[type(neuron_seed)] = [] for i in torch.arange(len(sub_seperator)): neuron_seed.clear() neuron_c = neuron_seed.clone() sub_dataset_i = sub_dataset_list[i] kwargs['data'] = sub_dataset_i kwargs['n_rules'] = int(n_rules_tree[j][i]) neuron_c.forward(**kwargs) w_sub.append(neuron_c.get_rules().consequent_list) rules_sub.append(neuron_c) rules_tree.append(rules_sub) else: # creat neural net work loss_fn = nn.CrossEntropyLoss() n_neuron = len(fea_seperator[0]) n_rules = int(n_rules_tree[0][0]) model_foot: ConsequentNet = ConsequentNet(n_neuron, n_rules) optimizer = torch.optim.Adam(model_foot.parameters(), lr=0.0001) epochs = 1500 train_losses = [] for epoch in range(epochs): model_foot.train() outputs = model_foot(w_sub) loss = loss_fn(outputs.double(), data.Y.long().squeeze(1)) loss.backward() optimizer.step() train_losses.append(loss.item()) print( f"epoch : {epoch + 1}, train loss : {train_losses[-1]} " ) # validate the model model_foot.eval() with torch.no_grad(): outputs = model_foot(w_sub) loss = loss_fn(outputs, data.Y.long().squeeze(1)) valid_losses = loss.item() _, predicted = torch.max(outputs.data, 1) correct = ( predicted == data.Y.squeeze().long()).sum().item() total = data.Y.size(0) accuracy = 100 * correct / total print( f"valid loss : {valid_losses}, valid acc : {accuracy}%" ) self.set_lei_net(model_foot) self.set_neuron_tree(rules_tree) return output
def forward(self, **kwargs): data: Dataset = kwargs['data'] if 'seperator' not in kwargs: seperator = FeaSeperator(data.name).get_seperator() else: seperator: FeaSeperator = kwargs['seperator'] fea_seperator = seperator.get_seperator() n_rules_tree = seperator.get_n_rule_tree() neuron_seed = self.get_neuron_seed() sub_seperator = fea_seperator[int(0)] sub_dataset_list = data.get_subset_fea(sub_seperator) sub_dataset_tmp = sub_dataset_list[0] n_rule_tmp = int(n_rules_tree[0][0]) n_smpl = sub_dataset_tmp.X.shape[0] n_fea_tmp = sub_dataset_tmp.X.shape[1] + 1 n_h = n_rule_tmp * n_fea_tmp h_all = torch.empty(0, n_smpl, n_h).double() n_branch = len(sub_seperator) # get neuron tree rules_tree: List[List[type(neuron_seed)]] = [] rules_sub: List[type(neuron_seed)] = [] # get output of every branches of upper dfnn layer for i in torch.arange(n_branch): neuron_seed.clear() neuron_c = neuron_seed.clone() sub_dataset_i = sub_dataset_list[i] kwargs['data'] = sub_dataset_i kwargs['n_rules'] = int(n_rules_tree[0][i]) neuron_c.forward(**kwargs) rules_sub.append(neuron_c) # get rules in neuron and update centers and bias rule_ao = neuron_c.get_rules() # get h computer in neuron h_computer_ao = neuron_c.get_h_computer() h_tmp, _ = h_computer_ao.comute_h(sub_dataset_i.X, rule_ao) h_cal_tmp = h_tmp.permute((1, 0, 2)) # N * n_rules * (d + 1) h_cal_tmp: torch.Tensor = h_cal_tmp.reshape(n_smpl, n_h) h_all = torch.cat((h_all, h_cal_tmp.unsqueeze(0)), 0) rules_tree.append(rules_sub) self.set_neuron_tree(rules_tree) # set consequent net consequent_net: ConsequentNet = ConsequentNet(n_branch, n_h) optimizer = torch.optim.Adam(consequent_net.parameters(), lr=0.001) loss_fn = nn.CrossEntropyLoss() # transform h tensor to torch dataset train_dataset = DatasetH(x=h_all, y=data.Y) train_loader = DataLoader(dataset=train_dataset, batch_size=n_smpl, shuffle=False) epochs = 3000 train_th = 0.000001 train_losses = [] for epoch in range(epochs): consequent_net.train() for i, (h_batch, labels) in enumerate(train_loader): optimizer.zero_grad() outputs = consequent_net(h_batch) loss = loss_fn(outputs.double(), labels.long().squeeze(1)) loss.backward() optimizer.step() train_losses.append(loss.item()) print(f"epoch : {epoch + 1}, train loss : {train_losses[-1]} ") # if len(train_losses) >= 2: # loss_diff = abs(train_losses[-1] - train_losses[len(train_losses)-2]) # # when the whole model stops to update # if loss_diff < train_th: # break # validate the model consequent_net.eval() correct = 0 total = 0 with torch.no_grad(): for i, (h_batch, labels) in enumerate(train_loader): outputs = consequent_net(h_batch) # loss = loss_fn(outputs, labels.long().squeeze(1)) # valid_losses = loss.item() _, predicted = torch.max(outputs.data, 1) correct += ( predicted == labels.squeeze().long()).sum().item() total += labels.size(0) # print(f"valid loss : {valid_losses}") accuracy = 100 * correct / total print(f"train acc : {accuracy}%") self.__consequent_net = consequent_net
def forward(self, **kwargs): train_data: Dataset = kwargs['train_data'] test_data: Dataset = kwargs['test_data'] if 'seperator' not in kwargs: seperator = FeaSeperator(train_data.name).get_seperator() else: seperator: FeaSeperator = kwargs['seperator'] para_mu = kwargs['para_mu'] para_mu1 = kwargs['para_mu1'] fea_seperator = seperator.get_seperator() seperator.generate_n_rule_tree(kwargs['n_rules']) n_rules_tree = seperator.get_n_rule_tree() neuron_seed = self.get_neuron_seed() sub_seperator = fea_seperator[int(0)] sub_dataset_list = train_data.get_subset_fea(sub_seperator) sub_dataset_tmp = sub_dataset_list[0] n_rule_tmp = int(n_rules_tree[0][0]) n_smpl = sub_dataset_tmp.X.shape[0] n_fea_tmp = sub_dataset_tmp.X.shape[1] + 1 n_h = n_rule_tmp * n_fea_tmp h_all = torch.empty(0, n_smpl, n_h).double() n_branch = len(sub_seperator) # get neuron tree rules_tree: List[List[type(neuron_seed)]] = [] rules_sub: List[type(neuron_seed)] = [] # get output of every branches of upper dfnn layer for i in torch.arange(n_branch): neuron_seed.clear() neuron_c = neuron_seed.clone() sub_dataset_i = sub_dataset_list[i] kwargs['data'] = sub_dataset_i kwargs['n_rules'] = int(n_rules_tree[0][i]) neuron_c.forward(**kwargs) rules_sub.append(neuron_c) # get rules in neuron and update centers and bias rule_ao = neuron_c.get_rules() # get h computer in neuron h_computer_ao = neuron_c.get_h_computer() h_tmp, _ = h_computer_ao.comute_h(sub_dataset_i.X, rule_ao) h_cal_tmp = h_tmp.permute((1, 0, 2)) # N * n_rules * (d + 1) h_cal_tmp: torch.Tensor = h_cal_tmp.reshape(n_smpl, n_h) h_all = torch.cat((h_all, h_cal_tmp.unsqueeze(0)), 0) rules_tree.append(rules_sub) self.set_neuron_tree(rules_tree) # set bottom level AO n_midle_output = kwargs['n_hidden_output'] w_x = torch.rand(n_branch, n_midle_output, n_h).double() w_y = torch.rand(n_branch * n_midle_output, 1).double() # start AO optimization diff = 1 loss = 100 train_loss_list = [] run_th = 0.00001 n_epoch = 20 run_epoch = 1 loss_test_old = 100 # get h_all h_brunch_all = torch.empty(n_smpl, 0).double() for i in torch.arange(n_branch): h_brunch = h_all[i, :, :].repeat(1, n_midle_output) h_brunch_all = torch.cat((h_brunch_all, h_brunch), 1) # load better parameters data_save_dir = f"./para_file/{kwargs['dataset_name']}" if not os.path.exists(data_save_dir): os.makedirs(data_save_dir) para_file = f"{data_save_dir}/ao_{kwargs['n_rules']}_{kwargs['n_hidden_output']}.pt" # if os.path.exists(para_fi."] g_w_x_best = None g_w_y_best = None # while diff > run_th and run_epoch < n_epoch: while run_epoch < n_epoch: # fix w_y update w_x w_y_brunch_all = w_y.repeat(1, n_h) w_y_brunch_all = w_y_brunch_all.reshape(1, -1) w_x_h_brunch = torch.mul(h_brunch_all, w_y_brunch_all) w_x_brunch = torch.inverse( w_x_h_brunch.t().mm(w_x_h_brunch) + para_mu * torch.eye(w_x_h_brunch.shape[1]).double()).mm( w_x_h_brunch.t().mm(train_data.Y)) w_x_cal = w_x_brunch.squeeze().reshape(n_branch, n_midle_output * n_h) w_x = w_x_cal.reshape(n_branch, n_midle_output, n_h) # fix w_x update w_y w_y_h_cal = torch.empty(n_smpl, 0).double() for i in torch.arange(n_branch): w_y_h_brunch = h_all[i, :, :].mm(w_x[i, :, :].t()).squeeze() if n_midle_output == 1: w_y_h_cal = torch.cat( (w_y_h_cal, w_y_h_brunch.unsqueeze(1)), 1) else: w_y_h_cal = torch.cat((w_y_h_cal, w_y_h_brunch), 1) w_y = torch.inverse(w_y_h_cal.t().mm(w_y_h_cal) + para_mu1 * torch.eye(w_y_h_cal.shape[1]).double()).mm( w_y_h_cal.t().mm(train_data.Y)) # compute loss y_tmp = train_data.Y y_hap_tmp = w_y_h_cal.mm(w_y) # loss_tmp = torch.norm(y_tmp - y_hap_tmp) loss_tmp = mean_squared_error(y_tmp, y_hap_tmp) diff = abs(loss_tmp - loss) loss = loss_tmp train_loss_list.append(loss) self.__w_x = w_x self.__w_y = w_y # snoop the test data performance test_y = self.predict(test_data, kwargs['seperator']) loss_train = mean_squared_error(y_tmp, y_hap_tmp) loss_test = mean_squared_error(test_y, test_data.Y) # print(f"Loss of test: {loss_test}") if loss_test < loss_test_old and loss_test >= loss_train: g_w_x_best = w_x g_w_y_best = w_y run_epoch = run_epoch + 1 # print(f"Loss of AO: {loss} w_y: {w_y.squeeze()[1:5]}") # x = torch.linspace(1, len(train_loss_list) + 1, len(train_loss_list)).numpy() # y = train_loss_list # plt.title('Result Analysis') # plt.plot(x, y, color='green', label='loss value') # # plt.plot(x, test_acys, color='red', label='training accuracy') # plt.legend() # 显示图例 # # plt.xlabel('iteration epochs') # plt.ylabel('loss value') # plt.show() self.__w_x = w_x self.__w_y = w_y if g_w_x_best is not None: self.__w_x = g_w_x_best self.__w_y = g_w_y_best para_dict = dict() para_dict["w_x"] = w_x para_dict["w_y"] = w_y torch.save(para_dict, para_file)