for idx, params in enumerate(HYPER_PARA): BATCH_SIZE, DROP_RT, LR, EPOCHS = params best_roc = [] best_prc = [] data_loader = DataLoader(train_data, batch_size=BATCH_SIZE, shuffle=True, collate_fn=collate) for run in range(1): model = GCN(NUM_ELEM, EMBEDDING_DIM, HIDDEN_DIM, EDGE_HIDDEN_DIM, NUM_CLS, NUM_LYS, ADD_ON_FEATS, F.relu, DROP_RT) loss_func = nn.CrossEntropyLoss() optimizer = optim.Adam(model.parameters(), lr=LR) device = torch.device('cuda:0') model.to(device) last_roc = -1 last_prc = -1 epochs_no_imprv = 0 for epoch in range(EPOCHS): model.train() epoch_loss = 0 batch = tqdm(data_loader) for bg, label in batch: optimizer.zero_grad() prediction = model(bg) # loss = torch.mean(F.cross_entropy(prediction, label, reduction='none')
v = torch.from_numpy(features[1]).to(device) feature = torch.sparse.FloatTensor(i.t(), v, features[2]).to(device) i = torch.from_numpy(supports[0]).long().to(device) v = torch.from_numpy(supports[1]).to(device) support = torch.sparse.FloatTensor(i.t(), v, supports[2]).float().to(device) print('x :', feature) print('sp:', support) num_features_nonzero = feature._nnz() feat_dim = feature.shape[1] net = GCN(feat_dim, num_classes, num_features_nonzero) net.to(device) optimizer = optim.Adam(net.parameters(), lr=args.learning_rate) net.train() for epoch in range(args.epochs): out = net((feature, support)) out = out[0] loss = masked_loss(out, train_label, train_mask) loss += args.weight_decay * net.l2_loss() acc = masked_acc(out, train_label, train_mask) optimizer.zero_grad() loss.backward() optimizer.step()
# hyper parameters epochs = 200 nfeat = features.shape[1] nhid = 16 nclasses = labels.max().item() + 1 dropout = 0.5 lr = 0.01 weight_decay = 5e-4 # device setting device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') # Model model = GCN(nfeat, nhid, nclasses, dropout) optimzer = optim.Adam(model.parameters(), lr=lr, weight_decay=weight_decay) # To device model = model.to(device) features = features.to(device) adj = adj.to(device) labels = labels.to(device) idx_train = idx_train.to(device) idx_val = idx_val.to(device) idx_test = idx_test.to(device) def train(epoch): t = time.time() model.train() optimzer.zero_grad()
# print Y_train ## Neural Model # Loss and Optimizer nComm = 2 nHid1 = 50 nHid2 = 20 nHid3 = 20 nHid4 = 10 nFeat = X_Tags_Feature.shape[1] #len(Tags_Features) gcn_model = GCN(nFeat, nHid1, nComm) #gcn_model = GCN(nFeat, nHid1, nHid2, nHid3, nHid4, nComm) #gcn_model.load_state_dict(torch.load('gcn_complete2.pkl')) criterion = nn.CrossEntropyLoss() gcn_optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, gcn_model.parameters()), lr=args.learning_rate, weight_decay=args.weight_decay) epoch_loss = [] epoch_accuracy = [] train_accuracy = [] epochs = [] correct = 0 try: for epoch in range(1, args.num_epochs + 1): train_acc = 0 length = 0 loss_train = 0
print("n_train:",n_train) #norm_adj = normalize(adj) i = torch.LongTensor([norm_adj.row, norm_adj.col]) v = torch.FloatTensor(norm_adj.data) norm_adj = torch.sparse.FloatTensor(i,v, adj.shape) features = torch.tensor(features, dtype=torch.float, requires_grad=False) y_train = torch.tensor(y_train, dtype=torch.long, requires_grad=False) y_val = torch.tensor(y_val, dtype=torch.long, requires_grad=False) y_test = torch.tensor(y_test, dtype=torch.long, requires_grad=False) load_from = False gcn = GCN(embsize, hidsize, nclass, weight_decay) if load_from: gcn = torch.load(load_from) optimizer = torch.optim.Adam(gcn.parameters(recurse=True), lr=lr, weight_decay=weight_decay) train_time = time.time() val_loss_pre = 1e9 val_acc_pre = 0 dec_time = 0 for epoch in range(args.epoch): t = time.time() gcn.train() optimizer.zero_grad() output = gcn(features, norm_adj) pred = output[train_mask] ans = torch.argmax(y_train[train_mask],dim=1) loss = F.cross_entropy(pred, ans) train_acc = cal_accuracy(output, y_train, train_mask) loss.backward()
class ModelRunner: def __init__(self, params, logger, data_logger=None, epochs_logger=None): self._logger = logger self._epoch_logger = epochs_logger self._data_logger = EmptyLogger() if data_logger is None else data_logger self._parameters = params self._lr = params["lr"] self._is_nni = params['is_nni'] self._device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu') self._mse_loss = self.weighted_mse_loss self._temporal_loss = self.weighted_mse_loss self.model = GCN(num_of_features=self._parameters["feature_matrices"][0].shape[1], hid_size=self._parameters["hid_size"], num_of_classes=self._parameters["number_of_classes"], activation=self._parameters["activation"], dropout=self._parameters["dropout"]) self.model = self.model.to(self._device) self.opt = self._parameters["optimizer"](self.model.parameters(), lr=self._parameters['lr'], weight_decay=self._parameters['weight_decay']) @property def logger(self): return self._logger @property def data_logger(self): return self._data_logger def weighted_mse_loss(self, pred, target, weights=None): if weights is None: return ((pred - target) ** 2).sum(dim=1).sum().to(self._device) elif self._parameters['loss_weights_type'] == 'sqrt(N/Nj)': weights = torch.tensor(weights).to(device=self._device, dtype=torch.float) b = (torch.sqrt((weights).sum() / weights) *(pred - target) ** 2).sum(dim=1).sum().to(self._device) return b elif self._parameters['loss_weights_type'] == '1/Njs': weights = torch.tensor(weights).to(device=self._device, dtype=torch.float) b = (torch.tensor(1. / weights) * (pred - target) ** 2).sum(dim=1).sum().to(self._device) return b def run(self): train_results_l = [] test_results = [] # train for epoch in range(self._parameters["epochs"]): train_results = self.train(epoch) train_results_l.append(train_results) if epoch == self._parameters["epochs"]-1: # for grid test_res = self.test(epoch) test_results.append(test_res) if self._is_nni: nni.report_intermediate_result(test_res["f1_score_macro"][-1]) else: print(self._parameters["it_num"], self._parameters["iterations"], epoch + 1, self._parameters["epochs"], self._parameters["lr"], self._parameters["dropout"], self._parameters["hid_size"], self._parameters["weight_decay"], self._parameters["temporal_pen"], train_results['loss'].item(), train_results['tempo_loss'].item(), train_results['loss'].item() + train_results['tempo_loss'].item(), train_results['f1_score_macro'][-1], train_results['f1_score_micro'][-1], test_res["loss"], test_res["tempo_loss"], test_res["loss"] + test_res["tempo_loss"], test_res["f1_score_macro"][-1], test_res["f1_score_micro"][-1]) self._logger.debug('Epoch: {:04d} '.format(epoch + 1) + 'lr: {:04f} '.format(self._parameters['lr']) + 'dropout: {:04f} '.format(self._parameters['dropout']) + 'hid_size: {:04f} '.format(self._parameters['hid_size']) + 'weight_decay: {:04f} '.format(self._parameters['weight_decay']) + 'temporal_pen: {:04f} '.format(self._parameters['temporal_pen']) + 'reg_loss_train: {:.4f} '.format(train_results['loss']) + 'temp_loss: {:.4f} '.format(train_results['tempo_loss'])) result = self.test('test', print_to_file=True) if self._is_nni: nni.report_final_result(result["f1_score_macro"]) return train_results_l, test_results, result, self._parameters def train(self, epoch): z_vals, outputs = [], [] labeled_indices = self._parameters['training_inds'] labels = self._parameters['training_labels'] tempo_loss = 0. loss_train = 0. self.model.train() self.opt.zero_grad() for idx, adj in enumerate(self._parameters["adj_matrices"]): input_features = torch.from_numpy(self._parameters["feature_matrices"][idx]).to(dtype=torch.float, device=self._device) z, output = self.model(input_features, adj) output = output[labeled_indices[idx], :] # Njs are the weights of the loss using the adj mx. # they should be either used or not. Nj_s = [sum([labels[u][t][j] for u in range(len(self._parameters["adj_matrices"])) for t in range(len(labels[u]))]) for j in range(self._parameters['number_of_classes'])] loss_train += self._mse_loss(output, labels[idx], Nj_s) # loss_train += self._mse_loss(output, labels[idx].float()) #without weights using the build-in mse z_vals.append(z) # After 1 GCN layer outputs.append(output) # Final predictions # counts the number of cross_year_persons z_appearances = 0. for t in range(len(z_vals) - 1): t_inds = self._parameters['training_inds'][t] t_plus_one_inds = self._parameters['training_inds'][t + 1] z_inds = [i for i in t_inds if i in t_plus_one_inds] z_appearances += len(z_inds) z_val_t = z_vals[t][z_inds, :] z_val_t_plus_1 = z_vals[t+1][z_inds, :] loss = self._temporal_loss(z_val_t_plus_1, z_val_t) tempo_loss += self._parameters["temporal_pen"] * loss tempo_loss /= z_appearances loss_train /= sum([len(labeled_indices[u]) for u in range(len(outputs))]) total_loss = loss_train + tempo_loss total_loss.backward() self.opt.step() f1_score_macro, f1_score_micro = [],[] if epoch == self._parameters['epochs']-1: for i in range(len(labels)): f1_mac, f1_mic, list_real, list_pred = self.accuracy_f1_score(outputs[i], labels[i]) f1_score_macro.append(f1_mac) f1_score_micro.append(f1_mic) result = {"loss": loss_train, "f1_score_macro": f1_score_macro, "f1_score_micro": f1_score_micro, "tempo_loss": tempo_loss} return result def test(self,epoch, print_to_file=False): z_vals, outputs = [], [] labeled_indices = self._parameters['test_inds'] labels = self._parameters['test_labels'] tempo_loss = 0. loss_test = 0. test_z_appearances = 0. self.model.eval() for idx, adj in enumerate(self._parameters["adj_matrices"]): test_mat = torch.from_numpy(self._parameters["feature_matrices"][idx]).to(self._device) z, output = self.model(*[test_mat, adj]) output = output[labeled_indices[idx], :] loss_test += self._mse_loss(output, labels[idx].float()) z_vals.append(z) outputs.append(output) if print_to_file: grid_outputs_folder = str(self._parameters['name']) self._logger.debug("\nprint to files") for i in range(len(self._parameters["adj_matrices"])): np_output = outputs[i].cpu().data.numpy() products_path = os.path.join(os.getcwd(),'dataset',self._parameters["dataset_name"], "gcn_outputs",grid_outputs_folder) if not os.path.exists(products_path): os.makedirs(products_path) with open(os.path.join("dataset",self._parameters["dataset_name"],"gcn_outputs",grid_outputs_folder, "gcn_" + str(i) + ".pkl"), "wb") as f: pickle.dump(np_output, f, protocol=pickle.HIGHEST_PROTOCOL) for t in range(len(z_vals) - 1): t_inds = self._parameters['test_inds'][t] t_plus_one_inds = self._parameters['test_inds'][t + 1] z_inds = [i for i in t_inds if i in t_plus_one_inds] test_z_appearances += len(z_inds) z_val_t = z_vals[t][z_inds, :] z_val_t_plus_1 = z_vals[t + 1][z_inds, :] tempo_loss += self._parameters["temporal_pen"] * self._temporal_loss(z_val_t_plus_1, z_val_t) tempo_loss /= test_z_appearances loss_test /= sum([len(labeled_indices[u]) for u in range(len(outputs))]) f1_score_macro, f1_score_micro = [], [] real,pred = [],[] if epoch == self._parameters['epochs'] - 1 or epoch == 'test': for i in range(len(labels)): #running over the years f1_mac, f1_mic, list_real, list_pred = self.accuracy_f1_score(outputs[i], labels[i]) f1_score_macro.append(f1_mac) f1_score_micro.append(f1_mic) real.extend(list_real) pred.extend(list_pred) self.confusion_matrix(real, pred) # of all years normalized to 1 for the last epoch test result = {"loss": loss_test.data.item(), "f1_score_macro": f1_score_macro, "f1_score_micro": f1_score_micro, "tempo_loss": tempo_loss.data.item()} return result def accuracy_f1_score(self,output, labels): pred, real = [],[] for person in range(labels.size(0)): #range of all persons for label in range(labels.size(1)): if labels[person,label]==0: continue else: argmax = output[person].max(0)[1] real.append(label) pred.append(argmax.cpu().item()) f1_macro = f1_score(real, pred, average='macro') f1_micro = f1_score(real, pred, average='micro') return f1_macro, f1_micro, real,pred def confusion_matrix(self, list_real, list_pred): matrix = np.zeros((self._parameters["number_of_classes"], self._parameters["number_of_classes"])) # classes X classes for i in range(len(list_pred)): matrix[list_real[i], list_pred[i]] += 1 row_sums = matrix.sum(axis=1, dtype='float') new_matrix = np.zeros((self._parameters["number_of_classes"], self._parameters["number_of_classes"])) # classes X classes for i, (row, row_sum) in enumerate(zip(matrix, row_sums)): if row_sum == 0: new_matrix[i, :] = 0 else: new_matrix[i, :] = row / row_sum new_matrix = np.around(new_matrix, 3) b = np.asarray(new_matrix) self._parameters['diag_sum'] = np.trace(b) self._parameters['diag_elements'] = np.diagonal(b) print('Diagonal (sum): ', np.trace(b)) print('Diagonal (elements): ', np.diagonal(b)) fig = plt.figure() ax = fig.add_subplot(111) cax = ax.matshow(new_matrix, interpolation='nearest') fig.colorbar(cax) ax.set_yticks(plt.np.arange(self._parameters["number_of_classes"])) ax.set_yticklabels(i for i in range(self._parameters["number_of_classes"])) ax.set_xticks(plt.np.arange(self._parameters["number_of_classes"])) ax.set_xticklabels(i for i in range(self._parameters["number_of_classes"])) ax.tick_params(axis='y', labelsize=7) ax.tick_params(axis='x', labelsize=7, labelbottom=True, labeltop=False) plt.title('Confusion matrix') ax.axis('image') plt.xlabel("Predicted label") plt.ylabel("Real label") mypath = "./dataset/"+self._parameters["dataset_name"]+"/figures" if not os.path.exists(mypath): os.makedirs(mypath) plt.savefig("./dataset/"+self._parameters["dataset_name"]+"/figures/cofution_matrix"+str(self._parameters['name'])+time.strftime("%Y%m%d_%H%M%S")+".png") plt.clf() plt.close() return
def train_classification_gcn(self, Adj, features, nfeats, labels, nclasses, train_mask, val_mask, test_mask, args): model = GCN(in_channels=nfeats, hidden_channels=args.hidden, out_channels=nclasses, num_layers=args.nlayers, dropout=args.dropout2, dropout_adj=args.dropout_adj2, Adj=Adj, sparse=args.sparse) optimizer = torch.optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.w_decay) bad_counter = 0 best_val = 0 best_model = None best_loss = 0 best_train_loss = 0 if torch.cuda.is_available(): model = model.cuda() train_mask = train_mask.cuda() val_mask = val_mask.cuda() test_mask = test_mask.cuda() features = features.cuda() labels = labels.cuda() for epoch in range(1, args.epochs + 1): model.train() loss, accu = self.get_loss_fixed_adj(model, train_mask, features, labels) optimizer.zero_grad() loss.backward() optimizer.step() if epoch % 10 == 0: model.eval() val_loss, accu = self.get_loss_fixed_adj( model, val_mask, features, labels) if accu > best_val: bad_counter = 0 best_val = accu best_model = copy.deepcopy(model) best_loss = val_loss best_train_loss = loss else: bad_counter += 1 if bad_counter >= args.patience: break print("Val Loss {:.4f}, Val Accuracy {:.4f}".format( best_loss, best_val)) best_model.eval() test_loss, test_accu = self.get_loss_fixed_adj(best_model, test_mask, features, labels) print("Test Loss {:.4f}, Test Accuracy {:.4f}".format( test_loss, test_accu)) return best_val, test_accu, best_model
def main(): print('settings: ', args) setup() rnd_state = np.random.RandomState(args.seed) print("Loading data ... ... ...") dataset = TUDataset('./data/%s/' % args.dataset, name=args.dataset, use_node_attr=args.use_cont_node_attr) train_ids, test_ids = split_ids(rnd_state.permutation(len(dataset)), folds=args.n_folds) print("Data loaded !!!") acc_folds = [] for fold_id in range(args.n_folds): loaders = [] for split in ['train', 'test']: gdata = dataset[torch.from_numpy( (train_ids if split.find('train') >= 0 else test_ids)[fold_id])] loader = DataLoader(gdata, batch_size=args.batch_size, shuffle=split.find('train') >= 0, num_workers=args.threads, collate_fn=collate_batch) loaders.append(loader) print('\nFOLD {}, train {}, test {}'.format(fold_id, len(loaders[0].dataset), len(loaders[1].dataset))) model = GCN(in_features=loaders[0].dataset.num_features, out_features=loaders[0].dataset.num_classes, n_hidden=args.n_hidden, filters=args.filters, dropout=args.dropout).to(args.device) print('\nInitialize model') print(model) train_params = list( filter(lambda p: p.requires_grad, model.parameters())) print('N trainable parameters:', np.sum([p.numel() for p in train_params])) optimizer = optim.Adam(train_params, lr=args.lr, weight_decay=args.wd, betas=(0.5, 0.999)) scheduler = lr_scheduler.MultiStepLR(optimizer, args.lr_decay_steps, gamma=0.1) for epoch in range(args.epochs): train(loaders[0], model=model, epoch=epoch, optimizer=optimizer, scheduler=scheduler) acc = test(loaders[1], model=model, epoch=epoch) acc_folds.append(acc) print(acc_folds) print('{}-fold cross validation avg acc (+- std): {} ({})'.format( args.n_folds, np.mean(acc_folds), np.std(acc_folds)))
def main(args): device = torch.device('cuda' if args.cuda else 'cpu') adj, features, labels, idx_train, idx_val, idx_test = load_data( args.dataset) if args.model == 'gcn': model = GCN(nfeat=features.size(1), nhid=args.hidden, nclass=labels.max().item() + 1, dropout=args.dropout) print('Model: GCN') elif args.model == 'gat': model = GAT(nfeat=features.size(1), nhid=args.hidden, nclass=labels.max().item() + 1, dropout=args.dropout, alpha=args.alpha, nheads=args.n_heads) print('Model: GAT') elif args.model == 'spgcn': model = SpGCN(nfeat=features.size(1), nhid=args.hidden, nclass=labels.max().item() + 1, dropout=args.dropout) print('Model: SpGCN') elif args.model == 'spgat': model = SpGAT(nfeat=features.size(1), nhid=args.hidden, nclass=labels.max().item() + 1, dropout=args.dropout, alpha=args.alpha, nheads=args.n_heads) print('Model: SpGAT') optimizer = optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.weight_decay) if args.cuda: adj = adj.cuda() features = features.cuda() labels = labels.cuda() idx_train = idx_train.cuda() idx_val = idx_val.cuda() idx_test = idx_test.cuda() model.cuda() print(device) def train(epoch): model.train() optimizer.zero_grad() output = model(features, adj) loss_train = F.nll_loss(output[idx_train], labels[idx_train]) acc_train = accuracy(output[idx_train], labels[idx_train]) loss_train.backward() optimizer.step() if not args.fastmode: model.eval() output = model(features, adj) loss_val = F.nll_loss(output[idx_val], labels[idx_val]) acc_val = accuracy(output[idx_val], labels[idx_val]) # print('Epoch: {:04d}'.format(epoch + 1), # 'loss_train: {:.4f}'.format(loss_train.item()), # 'acc_train: {:.4f}'.format(acc_train.item()), # 'loss_val: {:.4f}'.format(loss_val.item()), # 'acc_val: {:.4f}'.format(acc_val.item())) pbar.set_description( '| epoch: {:4d} | loss_train: {:.4f} | acc_train: {:.4f} |' ' loss_val: {:.4f} | acc_val: {:.4f}'.format( epoch + 1, loss_train.item(), acc_train.item(), loss_val.item(), acc_val.item())) return loss_train.item(), loss_val.item() def test(): model.eval() output = model(features, adj) loss_test = F.nll_loss(output[idx_test], labels[idx_test]) acc_test = accuracy(output[idx_test], labels[idx_test]) print("Test set results:", "loss= {:.4f}".format(loss_test.item()), "accuracy= {:.4f}".format(acc_test.item())) losses = {} pbar = tqdm(range(args.epochs)) for epoch in pbar: loss_train, loss_val = train(epoch) if epoch % 10 == 0: if len(losses) == 0: losses['train'] = [loss_train] losses['val'] = [loss_val] else: losses['train'].append(loss_train) losses['val'].append(loss_val) f, ax = plt.subplots() train_loss = ax.plot(losses['train'], label='Train Loss') val_loss = ax.plot(losses['val'], label='Validation Loss') ax.legend() ax.set_xlabel('Epoch / 10') ax.set_ylabel('Loss') plt.savefig('results/loss_{}_{}.png'.format(args.model, args.dataset), dpi=300) print('Optimization Finished!') test()
return loss_test.item(), acc_test.item() if __name__ == '__main__': random.seed(seed) rg = np.random.default_rng(seed) torch.manual_seed(seed) # Load data adj, features, labels, idx_train, idx_val, idx_test = load_data( dataset=dataset, seed=seed) # adj, features, labels, idx_train, idx_val, idx_test = load_rand_loadsplit_data(dataset) # Model and optimizer model = GCN(nfeat=features.shape[1], nhid=hidden, nclass=labels.max().item() + 1, dropout=dropout) optimizer = torch.optim.Adam(model.parameters(), lr=lr, weight_decay=weight_decay) # Train model t_total = time.time() for epoch in range(epochs): train(epoch) print("Optimization Finished!") print("Total time elapsed: {:.4f}s".format(time.time() - t_total)) # Testing res = test()
#!/usr/bin/env python3 # -*- coding: utf-8 -*- import matplotlib.pyplot as plt import torch import torch.nn.functional as F from model import GCN from graph_builder import build_karate_club_graph, draw import warnings warnings.filterwarnings("ignore") # Model net = GCN(34, 5, 2) optimizer = torch.optim.Adam(net.parameters(), lr=0.01) all_logits = [] print("Model structure:", net) # Data G = build_karate_club_graph() inputs = torch.eye(34) labeled_nodes = torch.tensor([0, 33]) labels = torch.tensor([0, 1]) # Train net.train() for epoch in range(30): optimizer.zero_grad()
def main(args): # 0. initial setting # set environmet cudnn.benchmark = True if not os.path.isdir(os.path.join(args.path, './ckpt')): os.mkdir(os.path.join(args.path, './ckpt')) if not os.path.isdir(os.path.join(args.path, './results')): os.mkdir(os.path.join(args.path, './results')) if not os.path.isdir(os.path.join(args.path, './ckpt', args.name)): os.mkdir(os.path.join(args.path, './ckpt', args.name)) if not os.path.isdir(os.path.join(args.path, './results', args.name)): os.mkdir(os.path.join(args.path, './results', args.name)) if not os.path.isdir(os.path.join(args.path, './results', args.name, "log")): os.mkdir(os.path.join(args.path, './results', args.name, "log")) # set logger logger = logging.getLogger() logger.setLevel(logging.INFO) formatter = logging.Formatter('%(asctime)s - %(message)s') handler = logging.FileHandler( os.path.join( args.path, "results/{}/log/{}.log".format( args.name, time.strftime('%c', time.localtime(time.time()))))) handler.setFormatter(formatter) logger.addHandler(handler) logger.addHandler(logging.StreamHandler()) args.logger = logger # set cuda if torch.cuda.is_available(): args.logger.info("running on cuda") args.device = torch.device("cuda") args.use_cuda = True else: args.logger.info("running on cpu") args.device = torch.device("cpu") args.use_cuda = False args.logger.info("[{}] starts".format(args.name)) # 1. load data adj, features, labels, idx_train, idx_val, idx_test = load_data() # 2. setup CORA_NODES = 2708 CORA_FEATURES = 1433 CORA_CLASSES = 7 CITESEER_NODES = 3327 CITESEER_FEATURES = 3703 CITESEER_CLASSES = 6 (num_nodes, feature_dim, classes) = (CORA_NODES, CORA_FEATURES, CORA_CLASSES) if args.dataset == 'cora' else ( CITESEER_NODES, CITESEER_FEATURES, CITESEER_CLASSES) args.logger.info("setting up...") model = GCN(args, feature_dim, args.hidden, classes, args.dropout) if args.model == 'gcn' else SpGAT( args, feature_dim, args.hidden, classes, args.dropout, args.alpha, args.n_heads) model.to(args.device) loss_fn = nn.NLLLoss() optimizer = optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.weight_decay) if args.load: loaded_data = load(args, args.ckpt) model.load_state_dict(loaded_data['model']) optimizer.load_state_dict(loaded_data['optimizer']) # 3. train / test if not args.test: # train args.logger.info("starting training") train_loss_meter = AverageMeter(args, name="Loss", save_all=True, x_label="epoch") val_acc_meter = AverageMeter(args, name="Val Acc", save_all=True, x_label="epoch") earlystop_listener = val_acc_meter.attach_combo_listener( (lambda prev, new: prev.max >= new.max), threshold=args.patience) steps = 1 for epoch in range(1, 1 + args.epochs): spent_time = time.time() model.train() train_loss_tmp_meter = AverageMeter(args) if args.start_from_step is not None: if steps < args.start_from_step: steps += 1 continue optimizer.zero_grad() batch = len(idx_train) output = model(features.to(args.device), adj.to(args.device)) loss = loss_fn(output[idx_train], labels[idx_train].to(args.device)) loss.backward() optimizer.step() train_loss_tmp_meter.update(loss, weight=batch) steps += 1 train_loss_meter.update(train_loss_tmp_meter.avg) spent_time = time.time() - spent_time args.logger.info( "[{}] train loss: {:.3f} took {:.1f} seconds".format( epoch, train_loss_tmp_meter.avg, spent_time)) model.eval() spent_time = time.time() if not args.fastmode: with torch.no_grad(): output = model(features.to(args.device), adj.to(args.device)) acc = accuracy(output[idx_val], labels[idx_val]) * 100.0 val_acc_meter.update(acc) earlystop = earlystop_listener.listen() spent_time = time.time() - spent_time args.logger.info( "[{}] val acc: {:2.1f} % took {:.1f} seconds".format( epoch, acc, spent_time)) if steps % args.save_period == 0: save(args, "epoch{}".format(epoch), {'model': model.state_dict()}) train_loss_meter.plot(scatter=False) val_acc_meter.plot(scatter=False) val_acc_meter.save() if earlystop: break else: # test args.logger.info("starting test") model.eval() with torch.no_grad(): model(features.to(args.device), adj.to(args.device)) acc = accuracy(output[idx_test], labels[idx_test]) * 100 logger.d("test acc: {:2.1f} % took {:.1f} seconds".format( acc, spent_time))
def main(args, print_fn=print): print_fn("Experiment arguments: {}".format(args)) if args.random_seed: torch.manual_seed(args.random_seed) else: torch.manual_seed(123) # Load dataset if args.dataset.startswith('ogbl'): graph, split_edge = load_ogb_dataset(args.dataset) else: raise NotImplementedError num_nodes = graph.num_nodes() # set gpu if args.gpu_id >= 0 and torch.cuda.is_available(): device = 'cuda:{}'.format(args.gpu_id) else: device = 'cpu' if args.dataset == 'ogbl-collab': # ogbl-collab dataset is multi-edge graph use_coalesce = True else: use_coalesce = False # Generate positive and negative edges and corresponding labels # Sampling subgraphs and generate node labeling features seal_data = SEALData(g=graph, split_edge=split_edge, hop=args.hop, neg_samples=args.neg_samples, subsample_ratio=args.subsample_ratio, use_coalesce=use_coalesce, prefix=args.dataset, save_dir=args.save_dir, num_workers=args.num_workers, print_fn=print_fn) node_attribute = seal_data.ndata['feat'] edge_weight = seal_data.edata['edge_weight'].float() train_data = seal_data('train') val_data = seal_data('valid') test_data = seal_data('test') train_graphs = len(train_data.graph_list) # Set data loader train_loader = GraphDataLoader(train_data, batch_size=args.batch_size, num_workers=args.num_workers) val_loader = GraphDataLoader(val_data, batch_size=args.batch_size, num_workers=args.num_workers) test_loader = GraphDataLoader(test_data, batch_size=args.batch_size, num_workers=args.num_workers) # set model if args.model == 'gcn': model = GCN(num_layers=args.num_layers, hidden_units=args.hidden_units, gcn_type=args.gcn_type, pooling_type=args.pooling, node_attributes=node_attribute, edge_weights=edge_weight, node_embedding=None, use_embedding=True, num_nodes=num_nodes, dropout=args.dropout) elif args.model == 'dgcnn': model = DGCNN(num_layers=args.num_layers, hidden_units=args.hidden_units, k=args.sort_k, gcn_type=args.gcn_type, node_attributes=node_attribute, edge_weights=edge_weight, node_embedding=None, use_embedding=True, num_nodes=num_nodes, dropout=args.dropout) else: raise ValueError('Model error') model = model.to(device) parameters = model.parameters() optimizer = torch.optim.Adam(parameters, lr=args.lr) loss_fn = BCEWithLogitsLoss() print_fn("Total parameters: {}".format(sum([p.numel() for p in model.parameters()]))) # train and evaluate loop summary_val = [] summary_test = [] for epoch in range(args.epochs): start_time = time.time() loss = train(model=model, dataloader=train_loader, loss_fn=loss_fn, optimizer=optimizer, device=device, num_graphs=args.batch_size, total_graphs=train_graphs) train_time = time.time() if epoch % args.eval_steps == 0: val_pos_pred, val_neg_pred = evaluate(model=model, dataloader=val_loader, device=device) test_pos_pred, test_neg_pred = evaluate(model=model, dataloader=test_loader, device=device) val_metric = evaluate_hits(args.dataset, val_pos_pred, val_neg_pred, args.hits_k) test_metric = evaluate_hits(args.dataset, test_pos_pred, test_neg_pred, args.hits_k) evaluate_time = time.time() print_fn("Epoch-{}, train loss: {:.4f}, hits@{}: val-{:.4f}, test-{:.4f}, " "cost time: train-{:.1f}s, total-{:.1f}s".format(epoch, loss, args.hits_k, val_metric, test_metric, train_time - start_time, evaluate_time - start_time)) summary_val.append(val_metric) summary_test.append(test_metric) # summary_val = np.array(summary_val) summary_test = np.array(summary_test) print_fn("Experiment Results:") print_fn("Best hits@{}: {:.4f}, epoch: {}".format(args.hits_k, np.max(summary_test), np.argmax(summary_test)))
seed = 9182 np.random.seed(seed) torch.manual_seed(seed) if torch.cuda.is_available(): torch.cuda.manual_seed(seed) # load data gcn_data = GCN_DATA(dataset, preprocess, validation, generalized, PREFINE_CLASS_path) # Model and optimizer model = GCN(input_dim=gcn_data.all_attributes.shape[1], hiddens_dim=hiddens_dim, output_dim=gcn_data.true_class_weight.shape[1], dropout=dropout) optimizer = optim.Adam(model.parameters(), lr=LR, weight_decay=weight_decay) loss_fun = torch.nn.MSELoss() if torch.cuda.is_available(): model = model.cuda() loss_fun = loss_fun.cuda() #为了分类,将one-hot表示的标签向量转化为索引表示 def map_label(labels): index_labels = np.where(labels.astype(int) == 1)[1] unique_class = np.unique(index_labels) class_num = len(unique_class) #print(unique_class,len(unique_class)) mapped_label = np.zeros((labels.shape[0], )) for i in range(class_num):
def train_gcn(training_features, training_adjs, training_labels, eval_features, eval_adjs, eval_labels, params, class_weights, activations, unary, coeffs, graph_params): device = torch.device( 'cuda:1') if torch.cuda.is_available() else torch.device('cpu') gcn = GCN(n_features=training_features[0].shape[1], hidden_layers=params["hidden_layers"], dropout=params["dropout"], activations=activations, p=graph_params["probability"], normalization=params["edge_normalization"]) gcn.to(device) opt = params["optimizer"](gcn.parameters(), lr=params["lr"], weight_decay=params["regularization"]) n_training_graphs = len(training_labels) graph_size = graph_params["vertices"] n_eval_graphs = len(eval_labels) counter = 0 # For early stopping min_loss = None for epoch in range(params["epochs"]): # -------------------------- TRAINING -------------------------- training_graphs_order = np.arange(n_training_graphs) np.random.shuffle(training_graphs_order) for i, idx in enumerate(training_graphs_order): training_mat = torch.tensor(training_features[idx], device=device) training_adj, training_lbs = map( lambda x: torch.tensor( data=x[idx], dtype=torch.double, device=device), [training_adjs, training_labels]) gcn.train() opt.zero_grad() output_train = gcn(training_mat, training_adj) output_matrix_flat = ( torch.mm(output_train, output_train.transpose(0, 1)) + 1 / 2).flatten() training_criterion = gcn_build_weighted_loss( unary, class_weights, training_lbs) loss_train = coeffs[0] * training_criterion(output_train.view(output_train.shape[0]), training_lbs) + \ coeffs[1] * gcn_pairwise_loss(output_matrix_flat, training_adj.flatten()) + \ coeffs[2] * gcn_binomial_reg(output_train, graph_params) loss_train.backward() opt.step() # -------------------------- EVALUATION -------------------------- graphs_order = np.arange(n_eval_graphs) np.random.shuffle(graphs_order) outputs = torch.zeros(graph_size * n_eval_graphs, dtype=torch.double) output_xs = torch.zeros(graph_size**2 * n_eval_graphs, dtype=torch.double) adj_flattened = torch.tensor( np.hstack([eval_adjs[idx].flatten() for idx in graphs_order])) for i, idx in enumerate(graphs_order): eval_mat = torch.tensor(eval_features[idx], device=device) eval_adj, eval_lbs = map( lambda x: torch.tensor( data=x[idx], dtype=torch.double, device=device), [eval_adjs, eval_labels]) gcn.eval() output_eval = gcn(eval_mat, eval_adj) output_matrix_flat = ( torch.mm(output_eval, output_eval.transpose(0, 1)) + 1 / 2).flatten() output_xs[i * graph_size**2:(i + 1) * graph_size**2] = output_matrix_flat.cpu() outputs[i * graph_size:(i + 1) * graph_size] = output_eval.view( output_eval.shape[0]).cpu() all_eval_labels = torch.tensor(np.hstack( [eval_labels[idx] for idx in graphs_order]), dtype=torch.double) eval_criterion = gcn_build_weighted_loss(unary, class_weights, all_eval_labels) loss_eval = ( coeffs[0] * eval_criterion(outputs, all_eval_labels) + coeffs[1] * gcn_pairwise_loss(output_xs, adj_flattened) + coeffs[2] * gcn_binomial_reg(outputs, graph_params)).item() if min_loss is None: current_min_loss = loss_eval else: current_min_loss = min(min_loss, loss_eval) if epoch >= 10 and params[ "early_stop"]: # Check for early stopping during training. if min_loss is None: min_loss = current_min_loss torch.save(gcn.state_dict(), "tmp_time.pt") # Save the best state. elif loss_eval < min_loss: min_loss = current_min_loss torch.save(gcn.state_dict(), "tmp_time.pt") # Save the best state. counter = 0 else: counter += 1 if counter >= 40: # Patience for learning break # After stopping early, our model is the one with the best eval loss. gcn.load_state_dict(torch.load("tmp_time.pt")) os.remove("tmp_time.pt") return gcn
args.cuda = not args.no_cuda and torch.cuda.is_available() #print(args.cuda) np.random.seed(args.seed) torch.manual_seed(args.seed) if args.cuda: torch.cuda.manual_seed(args.seed) adj,features,labels,idx_train,idx_val,idx_test = load_data() # Model and optimizer model = GCN(nfeat=features.shape[1], nhid=args.hidden, nclass=labels.max().item() + 1, dropout=args.dropout) optimizer = optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.weight_decay) adj,features,labels,idx_train,idx_val,idx_test = load_data() model = GCN(nfeat=features.shape[1],nhid = args.hidden,nclass=labels.max().item()+1,dropout=args.dropout) optimizer = optim.Adam(model.parameters(),lr = args.lr,weight_decay=args.weight_decay) if args.cuda: model.cuda() features = features.cuda() adj = adj.cuda() labels = labels.cuda()
if CD: net = net.cuda(0) batch_size = 8 start_epoch = 0 if True: ckpt = torch.load('/Disk5/junqi/CUB/early_skeleton_93.ckpt') net.load_state_dict(ckpt['net_state_dict']) start_epoch = ckpt['epoch'] + 1 datas = CUB() dataLoader = torch.utils.data.DataLoader(datas, batch_size=batch_size, shuffle=True, num_workers=4) optimizer = torch.optim.SGD(net.parameters(), lr=1e-2, momentum=0.9, weight_decay=1e-4) net.train() for epoch in range(start_epoch, 201): epoch_loss = 0 correct = 0 for iter, (inputs, mask, labels) in enumerate(dataLoader): if CD: inputs = inputs.cuda(0) mask = mask.cuda(0) labels = labels.cuda(0) prediction = net(G, inputs, mask) _, predict = torch.max(prediction, 1)
def main(opt): train_dataset = BADataset(opt.dataroot, opt.L, True, False, False) train_dataloader = BADataloader(train_dataset, batch_size=opt.batchSize, \ shuffle=True, num_workers=opt.workers, drop_last=True) valid_dataset = BADataset(opt.dataroot, opt.L, False, True, False) valid_dataloader = BADataloader(valid_dataset, batch_size=opt.batchSize, \ shuffle=True, num_workers=opt.workers, drop_last=True) test_dataset = BADataset(opt.dataroot, opt.L, False, False, True) test_dataloader = BADataloader(test_dataset, batch_size=opt.batchSize, \ shuffle=True, num_workers=opt.workers, drop_last=True) all_dataset = BADataset(opt.dataroot, opt.L, False, False, False) all_dataloader = BADataloader(all_dataset, batch_size=opt.batchSize, \ shuffle=False, num_workers=opt.workers, drop_last=False) opt.n_edge_types = train_dataset.n_edge_types opt.n_node = train_dataset.n_node net = GCN(opt, kernel_size=2, n_blocks=1, state_dim_bottleneck=opt.state_dim, annotation_dim_bottleneck=opt.annotation_dim) net.double() print(net) criterion = nn.BCELoss() if opt.cuda: net.cuda() criterion.cuda() optimizer = optim.Adam(net.parameters(), lr=opt.lr) early_stopping = EarlyStopping(patience=opt.patience, verbose=True) os.makedirs(OutputDir, exist_ok=True) train_loss_ls = [] valid_loss_ls = [] test_loss_ls = [] for epoch in range(0, opt.niter): train_loss = train(epoch, train_dataloader, net, criterion, optimizer, opt) valid_loss = valid(valid_dataloader, net, criterion, opt) test_loss = test(test_dataloader, net, criterion, opt) train_loss_ls.append(train_loss) valid_loss_ls.append(valid_loss) test_loss_ls.append(test_loss) early_stopping(valid_loss, net, OutputDir) if early_stopping.early_stop: print("Early stopping") break df = pd.DataFrame({ 'epoch': [i for i in range(1, len(train_loss_ls) + 1)], 'train_loss': train_loss_ls, 'valid_loss': valid_loss_ls, 'test_loss': test_loss_ls }) df.to_csv(OutputDir + '/loss.csv', index=False) net.load_state_dict(torch.load(OutputDir + '/checkpoint.pt')) inference(all_dataloader, net, criterion, opt, OutputDir)
loss_val = F.nll_loss(output[idx_val], labels[idx_val]) acc_val = accuracy(output[idx_val], labels[idx_val]) print('Epoch:{: 4d}\tloss_train:{:.4f}\tacc_train:{:.4f}\tloss_val:{:.4f}\tacc_val:{:.4f}'.format(epoch+1, loss, acc, loss_val, acc_val)) def test(model, adj, features, labels, idx_test): model.eval() output = model.forward(features, adj) loss = F.nll_loss(output[idx_test], labels[idx_test]) acc = accuracy(output[idx_test], labels[idx_test]) print('Test result\tloss:{:.4f}\tacc:{:.4f}'.format(loss, acc)) if __name__ == '__main__': seed = 2020 hidden_dim = 16 dropout = 0.5 learning_rate = 0.01 weight_decay = 5e-4 epochs = 200 set_seed(seed) adj, features, labels, idx_train, idx_val, idx_test = load_data() model = GCN(input_dim=features.shape[1], hidden_dim=hidden_dim, output_dim=labels.max().item()+1, dropout=dropout) optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=weight_decay) for e in range(epochs): train(e, model, optimizer, adj, features, labels, idx_train, idx_val) test(model, adj, features, labels, idx_test)