def __init__(self, args): super(modeler, self).__init__() self.args = args self.gcn = nn.ModuleList([GCN(args.ft_size, args.hid_units, args.activation, args.drop_prob, args.isBias) for _ in range(args.nb_graphs)]) self.disc = Discriminator(args.hid_units) self.H = nn.Parameter(torch.FloatTensor(1, args.nb_nodes, args.hid_units)) self.readout_func = self.args.readout_func if args.isAttn: self.attn = nn.ModuleList([Attention(args) for _ in range(args.nheads)]) if args.isSemi: self.logistic = LogReg(args.hid_units, args.nb_classes).to(args.device) self.init_weight()
def task(embeds): train_embs = embeds[0, idx_train] val_embs = embeds[0, idx_val] test_embs = embeds[0, idx_test] log = LogReg(hid_units, nb_classes) opt = torch.optim.Adam(log.parameters(), lr=0.01, weight_decay=0.0) if torch.cuda.is_available(): log.cuda() for _ in range(100): log.train() opt.zero_grad() logits = log(train_embs) loss = xent(logits, train_lbls) loss.backward() opt.step() logits = log(test_embs) preds = torch.argmax(logits, dim=1) acc = torch.sum(preds == test_lbls).float() / test_lbls.shape[0] return acc.detach().cpu().numpy()
embeds, _ = model.embed(features, sp_adj if sparse else adj, sparse, None) train_embs = embeds[0, idx_train] val_embs = embeds[0, idx_val] test_embs = embeds[0, idx_test] train_lbls = torch.argmax(labels[0, idx_train], dim=1) val_lbls = torch.argmax(labels[0, idx_val], dim=1) test_lbls = torch.argmax(labels[0, idx_test], dim=1) tot = torch.zeros(1) tot = tot.cuda() accs = [] for _ in range(50): log = LogReg(hid_units, nb_classes) opt = torch.optim.Adam(log.parameters(), lr=0.01, weight_decay=0.0) log.cuda() pat_steps = 0 best_acc = torch.zeros(1) best_acc = best_acc.cuda() for _ in range(100): log.train() opt.zero_grad() logits = log(train_embs) loss = xent(logits, train_lbls) loss.backward() opt.step()
args=parser.parse_args() n_classes = 10 n_epochs = 200 pre = Preprocessing('digits') pre.load_data(filename='train.csv', name='train') X_df = pre.get(name='train').drop(columns=['0']) y_df = pre.get(name='train')['0'] dtype = torch.float device = torch.device("cpu") model_name = 'logreg_digits' model = LogReg(model_name, 256, n_classes) learning_rate = 0.0001 batch_size = 32 train_classifier = TrainClassifier(model, X_df, y_df) trained_model , optimizer, criterion, loss_hist, loss_val_hist, best_param = train_classifier.run_train(n_epochs = n_epochs, lr=learning_rate, batch_size=batch_size) pre.save_results(loss_hist, loss_val_hist, f'{model_name}') trained_model.load_state_dict(state_dict=best_param) trained_model.eval() if args.s_model: m_exporter = ModelExporter('digits') m_exporter.save_nn_model(trained_model, optimizer, 0, n_classes, n_epochs, trained_model.get_args())
parser.add_argument('--n_feat', default=1000, help='number of features') parser.add_argument('--s_model', default=False, help='save trained model') args=parser.parse_args() pre = Preprocessing('IMDB') n_classes = 2 n_features = int(args.n_feat) n_epochs = 100 pre.load_data(filename=f'training_data_{n_features}.csv', name='training_data') X_df = pre.get(name='training_data').drop(columns=['target']) y_df = pre.get(name='training_data')['target'] model = LogReg('log_reg', n_features, n_classes) train_classifier = TrainClassifier(model, X_df, y_df) trained_model, optimizer, criterion, loss_hist, loss_validate_hist = train_classifier.run_train(n_epochs = n_epochs) pre.save_results(loss_hist, loss_validate_hist, f'log_reg_{100}') m_exporter = ModelExporter('IMDB') m_exporter.save_nn_model(trained_model, optimizer, n_features, n_classes, n_epochs) ##teeeeeest part pre.load_data(filename=f'test_data_{n_features}.csv', name='test_data') X_test_df = pre.get(name='test_data').drop(columns=['target']) y_test_df = pre.get(name='test_data')['target']
def run_GCN(args, gpu_id=None, exp_name=None, number=0, return_model=False, return_time_series=False): random.seed(args.seed) torch.cuda.manual_seed(args.seed) np.random.seed(args.seed) final_acc = 0 best_acc = 0 torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False running_device = "cpu" if gpu_id is None \ else torch.device('cuda:{}'.format(gpu_id) if torch.cuda.is_available() else 'cpu') dataset_kwargs = {} train_d, adj_list, x_list = get_dataset(args, dataset_kwargs) lable = train_d.data.y A_I = adj_list[0] A_I_nomal = adj_list[1] nb_edges = train_d.data.num_edges nb_nodes = train_d.data.num_nodes nb_feature = train_d.data.num_features nb_classes = int(lable.max() - lable.min()) + 1 lable_matrix = (lable.view(nb_nodes, 1).repeat(1, nb_nodes) == lable.view( 1, nb_nodes).repeat(nb_nodes, 1)) + 0 I = (torch.eye(nb_nodes).to(lable_matrix.device) == 1) lable_matrix[I] = 0 zero_vec = 0.0 * torch.ones_like(A_I_nomal) if args.dataset_name in [ 'Photo', 'DBLP', 'Crocodile', 'CoraFull', 'WikiCS' ]: useA = True else: useA = False model = UGRL_GCN_test(nb_nodes, nb_feature, args.dim, dim_x=args.dim_x, useact=args.usingact, liner=args.UsingLiner, dropout=args.dropout, useA=useA) optimiser = torch.optim.Adam(model.parameters(), lr=args.lr, weight_decay=0.0001) model.to(running_device) lable = lable.to(running_device) if args.dataset_name == 'WikiCS': train_lbls = lable[train_d.data.train_mask[:, args.NewATop]] # capture test_lbls = lable[train_d.data.test_mask] elif args.dataset_name in ['Cora', 'CiteSeer', 'PubMed']: train_lbls = lable[train_d.data.train_mask] test_lbls = lable[train_d.data.test_mask] elif args.dataset_name in ['Photo', 'DBLP', 'Crocodile', 'CoraFull']: train_index = [] test_index = [] for j in range(lable.max().item() + 1): num = ((lable == j) + 0).sum().item() index = torch.range(0, len(lable) - 1)[(lable == j)] x_list0 = random.sample(list(index), int(len(index) * 0.1)) for x in x_list0: train_index.append(int(x)) for c in range(len(lable)): if int(c) not in train_index: test_index.append(int(c)) train_lbls = lable[train_index] test_lbls = lable[test_index] val_lbls = lable[train_index] A_I_nomal_dense = A_I_nomal I_input = torch.eye(A_I_nomal.shape[1]) # .to(A_I_nomal.device) if args.dataset_name in ['PubMed', 'CoraFull', 'DBLP']: pass elif args.dataset_name in ['Crocodile', 'Photo', 'WikiCS']: A_I_nomal_dense = A_I_nomal_dense.to(running_device) ######################sparse################ if args.dataset_name in [ 'PubMed', 'Crocodile', 'CoraFull', 'DBLP', 'Photo', 'WikiCS' ]: A_I_nomal = A_I_nomal.to_sparse() model.sparse = True I_input = I_input.to_sparse() ######################sparse################ A_I_nomal = A_I_nomal.to(running_device) I_input = I_input.to(A_I_nomal.device) mask_I = I.to(running_device) zero_vec = zero_vec.to(running_device) my_margin = args.margin1 my_margin_2 = my_margin + args.margin2 margin_loss = torch.nn.MarginRankingLoss(margin=my_margin, reduce=False) num_neg = args.NN for current_iter, epoch in enumerate( tqdm(range(args.start_epoch, args.start_epoch + args.epochs + 1))): model.train() optimiser.zero_grad() idx = np.random.permutation(nb_nodes) feature_X = x_list[0].to(running_device) lbl_z = torch.tensor([0.]).to(running_device) feature_a = feature_X feature_p = feature_X feature_n = [] idx_list = [] idx_lable = [] for i in range(num_neg): idx_0 = np.random.permutation(nb_nodes) idx_list.append(idx_0) idx_lable.append(lable[idx_0]) feature_temp = feature_X[idx_0] feature_n.append(feature_temp) h_a, h_p, h_n_lsit, h_a_0, h_p_0, h_n_0_list = model(feature_a, feature_p, feature_n, A_I_nomal, I=I_input) s_p = F.pairwise_distance(h_a, h_p) cos_0_list = [] for h_n_0 in h_n_0_list: cos_0 = F.pairwise_distance(h_a_0, h_n_0) cos_0_list.append(cos_0) cos_0_stack = torch.stack(cos_0_list).detach() cos_0_min = cos_0_stack.min(dim=0)[0] cos_0_max = cos_0_stack.max(dim=0)[0] gap = cos_0_max - cos_0_min weight_list = [] for i in range(cos_0_stack.size()[0]): weight_list.append((cos_0_stack[i] - cos_0_min) / gap) s_n_list = [] s_n_cosin_list = [] for h_n in h_n_lsit: if args.dataset_name in ['Cora', 'CiteSeer']: s_n_cosin_list.append(cosine_dist(h_a, h_n)[mask_I].detach()) s_n = F.pairwise_distance(h_a, h_n) s_n_list.append(s_n) margin_label = -1 * torch.ones_like(s_p) loss_mar = 0 mask_margin_N = 0 i = 0 for s_n in s_n_list: loss_mar += (margin_loss(s_p, s_n, margin_label) * weight_list[i]).mean() mask_margin_N += torch.max((s_n - s_p.detach() - my_margin_2), lbl_z).sum() i += 1 mask_margin_N = mask_margin_N / num_neg string_1 = " loss_1: {:.3f}||loss_2: {:.3f}||".format( loss_mar.item(), mask_margin_N.item()) loss = loss_mar * args.w_loss1 + mask_margin_N * args.w_loss2 / nb_nodes if args.dataset_name in ['Cora']: loss = loss_mar * args.w_loss1 + mask_margin_N * args.w_loss2 loss.backward() optimiser.step() model.eval() if args.dataset_name in ['Crocodile', 'WikiCS', 'Photo']: h_p_d = h_p.detach() S_new = cosine_dist(h_p_d, h_p_d) model.A = normalize_graph(torch.mul(S_new, A_I_nomal_dense)).to_sparse() elif args.dataset_name in ['Cora', 'CiteSeer']: h_a, h_p = model.embed(feature_a, feature_p, feature_n, A_I_nomal, I=I_input) s_a = cosine_dist(h_a, h_a).detach() S = (torch.stack(s_n_cosin_list).mean(dim=0).expand_as(A_I) - s_a).detach() # zero_vec = -9e15 * torch.ones_like(S) one_vec = torch.ones_like(S) s_a = torch.where(A_I_nomal > 0, one_vec, zero_vec) attention = torch.where(S < 0, s_a, zero_vec) attention_N = normalize_graph(attention) attention[I] = 0 model.A = attention_N if epoch % 50 == 0: model.eval() h_a, h_p = model.embed(feature_a, feature_p, feature_n, A_I_nomal, I=I_input) if args.useNewA: embs = h_p #torch.cat((h_a,h_p),dim=1) else: embs = h_a if args.dataset_name in ['Cora', 'CiteSeer', 'PubMed']: embs = embs / embs.norm(dim=1)[:, None] if args.dataset_name == 'WikiCS': train_embs = embs[train_d.data.train_mask[:, args.NewATop]] test_embs = embs[train_d.data.test_mask] elif args.dataset_name in ['Cora', 'CiteSeer', 'PubMed']: train_embs = embs[train_d.data.train_mask] test_embs = embs[train_d.data.test_mask] # #val_embs = embs[train_d.data.val_mask] elif args.dataset_name in [ 'Photo', 'DBLP', 'Crocodile', 'CoraFull' ]: train_embs = embs[train_index] test_embs = embs[test_index] #val_embs = embs[train_index] accs = [] accs_small = [] xent = nn.CrossEntropyLoss() for _ in range(2): log = LogReg(args.dim, nb_classes) opt = torch.optim.Adam(log.parameters(), lr=1e-2, weight_decay=args.wd) log.to(running_device) for _ in range(args.num1): log.train() opt.zero_grad() logits = log(train_embs) loss = xent(logits, train_lbls) loss.backward() opt.step() logits = log(test_embs) preds = torch.argmax(logits, dim=1) acc = torch.sum( preds == test_lbls).float() / test_lbls.shape[0] accs.append(acc * 100) ac = [] for i in range(nb_classes): acc_small = torch.sum( preds[test_lbls == i] == test_lbls[test_lbls == i] ).float() / test_lbls[test_lbls == i].shape[0] ac.append(acc_small * 100) accs_small = ac accs = torch.stack(accs) string_3 = "" for i in range(nb_classes): string_3 = string_3 + "|{:.1f}".format(accs_small[i].item()) string_2 = Fore.GREEN + " epoch: {},accs: {:.1f},std: {:.2f} ".format( epoch, accs.mean().item(), accs.std().item()) tqdm.write(string_1 + string_2 + string_3) final_acc = accs.mean().item() best_acc = max(best_acc, final_acc) return final_acc, best_acc
def evaluate(embeds, idx_train, idx_val, idx_test, labels, device, isTest=True): hid_units = embeds.shape[2] nb_classes = labels.shape[2] xent = nn.CrossEntropyLoss() train_embs = embeds[0, idx_train] val_embs = embeds[0, idx_val] test_embs = embeds[0, idx_test] train_lbls = torch.argmax(labels[0, idx_train], dim=1) val_lbls = torch.argmax(labels[0, idx_val], dim=1) test_lbls = torch.argmax(labels[0, idx_test], dim=1) accs = [] micro_f1s = [] macro_f1s = [] macro_f1s_val = [] ## for _ in range(50): log = LogReg(hid_units, nb_classes) opt = torch.optim.Adam(log.parameters(), lr=0.01, weight_decay=0.0) log.to(device) val_accs = []; test_accs = [] val_micro_f1s = []; test_micro_f1s = [] val_macro_f1s = []; test_macro_f1s = [] for iter_ in range(50): # train log.train() opt.zero_grad() logits = log(train_embs) loss = xent(logits, train_lbls) loss.backward() opt.step() # val logits = log(val_embs) preds = torch.argmax(logits, dim=1) val_acc = torch.sum(preds == val_lbls).float() / val_lbls.shape[0] val_f1_macro = f1_score(val_lbls.cpu(), preds.cpu(), average='macro') val_f1_micro = f1_score(val_lbls.cpu(), preds.cpu(), average='micro') val_accs.append(val_acc.item()) val_macro_f1s.append(val_f1_macro) val_micro_f1s.append(val_f1_micro) # test logits = log(test_embs) preds = torch.argmax(logits, dim=1) test_acc = torch.sum(preds == test_lbls).float() / test_lbls.shape[0] test_f1_macro = f1_score(test_lbls.cpu(), preds.cpu(), average='macro') test_f1_micro = f1_score(test_lbls.cpu(), preds.cpu(), average='micro') test_accs.append(test_acc.item()) test_macro_f1s.append(test_f1_macro) test_micro_f1s.append(test_f1_micro) max_iter = val_accs.index(max(val_accs)) accs.append(test_accs[max_iter]) max_iter = val_macro_f1s.index(max(val_macro_f1s)) macro_f1s.append(test_macro_f1s[max_iter]) macro_f1s_val.append(val_macro_f1s[max_iter]) ### max_iter = val_micro_f1s.index(max(val_micro_f1s)) micro_f1s.append(test_micro_f1s[max_iter]) if isTest: print("\t[Classification] Macro-F1: {:.4f} ({:.4f}) | Micro-F1: {:.4f} ({:.4f})".format(np.mean(macro_f1s), np.std(macro_f1s), np.mean(micro_f1s), np.std(micro_f1s))) else: return np.mean(macro_f1s_val), np.mean(macro_f1s) test_embs = np.array(test_embs.cpu()) test_lbls = np.array(test_lbls.cpu()) run_kmeans(test_embs, test_lbls, nb_classes) run_similarity_search(test_embs, test_lbls)
def main(): saved_graph = os.path.join('assets', 'saved_graphs', 'best_dgi.pickle') saved_logreg = os.path.join('assets', 'saved_graphs', 'best_logreg.pickle') dataset = 'cora' # training params batch_size = 1 nb_epochs = 10000 patience = 25 lr = 0.001 l2_coef = 0.0 drop_prob = 0.0 hid_units = 512 sparse = True nonlinearity = 'prelu' # special name to separate parameters adj, features, labels, idx_train, idx_test, idx_val = process.load_data(dataset) features, _ = process.preprocess_features(features) nb_nodes = features.shape[0] ft_size = features.shape[1] nb_classes = labels.shape[1] adj = process.normalize_adj(adj + sp.eye(adj.shape[0])) if sparse: adj = process.sparse_mx_to_torch_sparse_tensor(adj) else: adj = (adj + sp.eye(adj.shape[0])).todense() features = torch.FloatTensor(features[np.newaxis]) if not sparse: adj = torch.FloatTensor(adj[np.newaxis]) labels = torch.FloatTensor(labels[np.newaxis]) idx_train = torch.LongTensor(idx_train) idx_val = torch.LongTensor(idx_val) idx_test = torch.LongTensor(idx_test) print("Training Nodes: {}, Testing Nodes: {}, Validation Nodes: {}".format(len(idx_train), len(idx_test), len(idx_val))) model = DGI(ft_size, hid_units, nonlinearity) optimiser = torch.optim.Adam(model.parameters(), lr=lr, weight_decay=l2_coef) if torch.cuda.is_available(): print('Using CUDA') model.cuda() features = features.cuda() if sparse: sp_adj = sp_adj.cuda() else: adj = adj.cuda() labels = labels.cuda() idx_train = idx_train.cuda() idx_val = idx_val.cuda() idx_test = idx_test.cuda() b_xent = nn.BCEWithLogitsLoss() xent = nn.CrossEntropyLoss() cant_wait = 0 best = 1e9 best_t = 0 if not os.path.exists(saved_graph): pbar = trange(nb_epochs) for epoch in pbar: model.train() optimiser.zero_grad() idx = np.random.permutation(nb_nodes) shuf_fts = features[:, idx, :] lbl_1 = torch.ones(batch_size, nb_nodes) lbl_2 = torch.zeros(batch_size, nb_nodes) lbl = torch.cat((lbl_1, lbl_2), 1) if torch.cuda.is_available(): shuf_fts = shuf_fts.cuda() lbl = lbl.cuda() logits = model(features, shuf_fts, adj, sparse, None, None, None) loss = b_xent(logits, lbl) pbar.desc = 'Loss: {:.4f}'.format(loss) if loss < best: best = loss best_t = epoch cnt_wait = 0 torch.save(model.state_dict(), saved_graph) else: cant_wait += 1 if cant_wait == patience: tqdm.write('Early stopping!') break loss.backward() optimiser.step() print('Loading {}th Epoch'.format(best_t) if best_t else 'Loading Existing Graph') model.load_state_dict(torch.load(saved_graph)) embeds, _ = model.embed(features, adj, sparse, None) train_embs = embeds[0, idx_train] val_embs = embeds[0, idx_val] test_embs = embeds[0, idx_test] train_lbls = torch.argmax(labels[0, idx_train], dim=1) val_lbls = torch.argmax(labels[0, idx_val], dim=1) test_lbls = torch.argmax(labels[0, idx_test], dim=1) tot = torch.zeros(1) if torch.cuda.is_available(): tot = tot.cuda() accs = [] print("\nValidation:") pbar = trange(50) for _ in pbar: log = LogReg(hid_units, nb_classes) opt = torch.optim.Adam(log.parameters(), lr=0.01, weight_decay=0.0) pat_steps = 0 best_acc = torch.zeros(1) if torch.cuda.is_available(): log.cuda() best_acc = best_acc.cuda() for _ in range(100): log.train() opt.zero_grad() logits = log(train_embs) loss = xent(logits, train_lbls) loss.backward() opt.step() logits = log(test_embs) preds = torch.argmax(logits, dim=1) acc = torch.sum(preds == test_lbls).float() / test_lbls.shape[0] accs.append(acc * 100) pbar.desc = "Accuracy: {:.2f}%".format(100 * acc) tot += acc torch.save(log.state_dict(), saved_logreg) accs = torch.stack(accs) print('Average Accuracy: {:.2f}%'.format(accs.mean())) print('Standard Deviation: {:.3f}'.format(accs.std())) print("\nTesting") logits = log(val_embs) preds = torch.argmax(logits, dim=1) acc = torch.sum(preds == val_lbls).float() / val_lbls.shape[0] print("Accuracy: {:.2f}%".format(100 * acc))
pre.get('train')['0'], test_size=0.01) #transfom to torch striuctures dtype = torch.float device = torch.device("cpu") X_train = torch.tensor(X_train_df.values, device=device, dtype=dtype) y_train = torch.tensor(y_train_df.values, device=device, dtype=dtype) # Softmax regression model n_features = X_train.size()[1] n_classes = len(np.unique(y_train.round().numpy())) model = LogReg(name='logreg', d_in=n_features, d_out=n_classes) criterion = nn.CrossEntropyLoss() optimizer = torch.optim.SGD(model.parameters(), lr=0.001, momentum=0.9) size_batch = 100 loss_hist = [] # Train start = time.time() for t in range(5000): X_train_mini, y_train_mini = get_mini_batching(X_train, y_train, size_batch) # Berechne die Vorhersage (foward step)
def process_inductive(dataset, gnn_type="GCNConv", K=None, random_init=False, runs=10): hyperparameters = get_hyperparameters() nb_epochs = hyperparameters["nb_epochs"] patience = hyperparameters["patience"] lr = hyperparameters["lr"] l2_coef = hyperparameters["l2_coef"] drop_prob = hyperparameters["drop_prob"] hid_units = hyperparameters["hid_units"] nonlinearity = hyperparameters["nonlinearity"] batch_size = hyperparameters["batch_size"] norm_features = torch_geometric.transforms.NormalizeFeatures() dataset_train = PPI( "./geometric_datasets/"+dataset, split="train", transform=norm_features, ) print(dataset_train) dataset_val = PPI( "./geometric_datasets/"+dataset, split="val", transform=norm_features, ) print(dataset_val) dataset_test = PPI( "./geometric_datasets/"+dataset, split="test", transform=norm_features, ) data = [] for d in dataset_train: data.append(d) for d in dataset_val: data.append(d) ft_size = dataset_train[0].x.shape[1] nb_classes = dataset_train[0].y.shape[1] # multilabel b_xent = nn.BCEWithLogitsLoss() loader_train = DataLoader( data, batch_size=hyperparameters["batch_size"], shuffle=True, ) loader_test = DataLoader( dataset_test, batch_size=hyperparameters["batch_size"], shuffle=False ) all_accs = [] for _ in range(runs): model = DGI(ft_size, hid_units, nonlinearity, update_rule=gnn_type, batch_size=1, K=K) model_name = get_model_name(dataset, gnn_type, K, random_init=random_init) print(model) optimiser = torch.optim.Adam(model.parameters(), lr=lr, weight_decay=l2_coef) if torch.cuda.is_available(): print('Using CUDA') model = model.cuda() model.train() torch.cuda.empty_cache() for epoch in range(20): if random_init: break total_loss = 0 batch_id = 0 model.train() loaded = list(loader_train) for batch in loaded: optimiser.zero_grad() if torch.cuda.is_available: batch = batch.to('cuda') nb_nodes = batch.x.shape[0] features = batch.x labels = batch.y edge_index = batch.edge_index idx = np.random.randint(0, len(data)) while idx == batch_id: idx = np.random.randint(0, len(data)) shuf_fts = torch.nn.functional.dropout(loaded[idx].x, drop_prob) edge_index2 = loaded[idx].edge_index lbl_1 = torch.ones(nb_nodes) lbl_2 = torch.zeros(shuf_fts.shape[0]) lbl = torch.cat((lbl_1, lbl_2), 0) if torch.cuda.is_available(): shuf_fts = shuf_fts.cuda() if edge_index2 is not None: edge_index2 = edge_index2.cuda() lbl = lbl.cuda() logits = model(features, shuf_fts, edge_index, batch=batch.batch, edge_index_alt=edge_index2) loss = b_xent(logits, lbl) loss.backward() optimiser.step() batch_id += 1 total_loss += loss.item() print(epoch, 'Train Loss:', total_loss/(len(dataset_train))) torch.save(model.state_dict(), './trained_models/'+model_name) torch.cuda.empty_cache() print('Loading last epoch') if not random_init: model.load_state_dict(torch.load('./trained_models/'+model_name)) model.eval() b_xent_reg = nn.BCEWithLogitsLoss(pos_weight=torch.tensor(2.25)) train_embs, whole_train_data = preprocess_embeddings(model, dataset_train) val_embs, whole_val_data = preprocess_embeddings(model, dataset_val) test_embs, whole_test_data = preprocess_embeddings(model, dataset_test) for _ in range(50): log = LogReg(hid_units, nb_classes) opt = torch.optim.Adam(log.parameters(), lr=0.01, weight_decay=0.0) log.cuda() pat_steps = 0 best = 1e9 log.train() for _ in range(250): opt.zero_grad() logits = log(train_embs) loss = b_xent_reg(logits, whole_train_data.y) loss.backward() opt.step() log.eval() val_logits = log(val_embs) loss = b_xent_reg(val_logits, whole_val_data.y) if loss.item() < best: best = loss.item() pat_steps = 0 if pat_steps >= 5: break pat_steps += 1 log.eval() logits = log(test_embs) preds = torch.sigmoid(logits) > 0.5 f1 = sklearn.metrics.f1_score(whole_test_data.y.cpu(), preds.long().cpu(), average='micro') all_accs.append(float(f1)) print() print('Micro-averaged f1:', f1) all_accs = torch.tensor(all_accs) with open("./results/"+model_name[:-4]+"_results.txt", "w") as f: f.writelines([str(all_accs.mean().item())+'\n', str(all_accs.std().item())]) print(all_accs.mean()) print(all_accs.std())
tot = torch.zeros(1) if args.cuda: tot = tot.cuda() tot_mac = 0 accs = [] mac_f1 = [] for _ in range(5): bad_counter = 0 best = 10000 loss_values = [] best_epoch = 0 train_patience = 50 log = LogReg(nout, nb_classes) opt = torch.optim.Adam(log.parameters(), lr=0.01, weight_decay=0.0) if args.cuda: log.cuda() for epoch in range(100000): log.train() opt.zero_grad() logits = log(train_embs) loss = xent(logits, train_lbls) logits_val = log(val_embs) loss_val = xent(logits_val, val_lbls) loss_values.append(loss_val) # print("train_loss: "+ str(loss) +" "+"val_loss: "+ str(loss_val) ) loss.backward() opt.step()
def process_transductive(dataset, gnn_type='GCNConv', K=None, random_init=False, runs=10, drop_sigma=False, just_plot=False): dataset_str = dataset norm_features = torch_geometric.transforms.NormalizeFeatures() dataset = Planetoid("./geometric_datasets"+'/'+dataset, dataset, transform=norm_features)[0] # training params batch_size = 1 # Transductive setting hyperparameters = get_hyperparameters() nb_epochs = hyperparameters["nb_epochs"] patience = hyperparameters["patience"] lr = hyperparameters["lr"] xent = nn.CrossEntropyLoss() l2_coef = hyperparameters["l2_coef"] drop_prob = hyperparameters["drop_prob"] hid_units = hyperparameters["hid_units"] nonlinearity = hyperparameters["nonlinearity"] nb_nodes = dataset.x.shape[0] ft_size = dataset.x.shape[1] nb_classes = torch.max(dataset.y).item()+1 # 0 based cnt features = dataset.x labels = dataset.y edge_index = dataset.edge_index edge_index, _ = torch_geometric.utils.add_remaining_self_loops(edge_index) mask_train = dataset.train_mask mask_val = dataset.val_mask mask_test = dataset.test_mask model_name = get_model_name(dataset_str, gnn_type, K, random_init=random_init, drop_sigma=drop_sigma) with open("./results/"+model_name[:-4]+"_results.txt", "w") as f: pass accs = [] for i in range(runs): model = DGI(ft_size, hid_units, nonlinearity, update_rule=gnn_type, K=K, drop_sigma=drop_sigma) print(model, model_name, drop_sigma) optimiser = torch.optim.Adam(model.parameters(), lr=lr) if torch.cuda.is_available(): print('Using CUDA') features = features.cuda() labels = labels.cuda() edge_index = edge_index.cuda() mask_train = mask_train.cuda() mask_val = mask_val.cuda() mask_test = mask_test.cuda() model = model.cuda() best_t = train_transductive(dataset, dataset_str, edge_index, gnn_type, model_name, K=K, random_init=random_init, drop_sigma=drop_sigma) xent = nn.CrossEntropyLoss() print('Loading {}th epoch'.format(best_t)) print(model, model_name) if not random_init: model.load_state_dict(torch.load('./trained_models/'+model_name)) model.eval() embeds, _ = model.embed(features, edge_index, None, standardise=False) if just_plot: plot_tsne(embeds, labels, model_name) exit(0) train_embs = embeds[mask_train, :] val_embs = embeds[mask_val, :] test_embs = embeds[mask_test, :] train_lbls = labels[mask_train] val_lbls = labels[mask_val] test_lbls = labels[mask_test] tot = torch.zeros(1) tot = tot.cuda() for _ in range(50): log = LogReg(hid_units, nb_classes) opt = torch.optim.Adam(log.parameters(), lr=0.01, weight_decay=0.0) log.cuda() pat_steps = 0 best_acc = torch.zeros(1) best_acc = best_acc.cuda() for _ in range(150): log.train() opt.zero_grad() logits = log(train_embs) loss = xent(logits, train_lbls) loss.backward() opt.step() logits = log(test_embs) preds = torch.argmax(logits, dim=1) acc = torch.sum(preds == test_lbls).float() / test_lbls.shape[0] accs.append(acc * 100) print(acc) tot += acc print('Average accuracy:', tot / 50) all_accs = torch.stack(accs, dim=0) with open("./results/"+model_name[:-4]+"_results.txt", "a+") as f: f.writelines([str(all_accs.mean().item())+'\n', str(all_accs.std().item())+'\n']) print(all_accs.mean()) print(all_accs.std())
def model_gen_fun(): model = LogReg(input_dim=32 * 32, num_classes=1).eval() return model
print(epoch, loss.item(), torch.max(norm_cut), torch.min(norm_cut)) embeddings = logits.detach() X_train = embeddings[idx_train] Y_train = labels[idx_train] X_test = embeddings[idx_test] Y_test = labels[idx_test] X_val = embeddings[idx_val] Y_val = labels[idx_val] tot = torch.zeros(1) tot = tot.cuda() totv = torch.zeros(1) totv = totv.cuda() accs = [] accsv = [] for _ in range(50): log = LogReg(args.embed_size, nb_classes) opt = torch.optim.Adam(log.parameters(), lr=0.01, weight_decay=0.0) log.cuda() pat_steps = 0 best_acc = torch.zeros(1) best_acc = best_acc.cuda() for _ in range(100): log.train() opt.zero_grad() logits = log(X_train) loss = xent(logits, Y_train) loss.backward() opt.step()
n_classes = 10 n_epochs = 200 pre = Preprocessing('digits') pre.load_data(filename='train_encoded.csv', name='train') X_df = pre.get(name='train').drop(columns=['0']) y_df = pre.get(name='train')['0'] dtype = torch.float device = torch.device("cpu") print(len(X_df.columns)) model_name = 'logreg_digits_encoded' model = LogReg(model_name, len(X_df.columns), n_classes) learning_rate = 0.0001 batch_size = 32 train_classifier = TrainClassifier(model, X_df, y_df) trained_model, optimizer, criterion, loss_hist, loss_val_hist, best_param = train_classifier.run_train( n_epochs=n_epochs, lr=learning_rate, batch_size=batch_size) pre.save_results(loss_hist, loss_val_hist, f'{model_name}') trained_model.load_state_dict(state_dict=best_param) trained_model.eval() if args.s_model: m_exporter = ModelExporter('digits') m_exporter.save_nn_model(trained_model, optimizer, 0, n_classes,
y_val = torch.tensor(y_val_df.values, device=device, dtype=dtype) # Softmax regression model #n_features = X.size()[1] n_classes = 2 H_0 = 150 H_1 = 125 n_iter = 500 print('features: ', n_features) print('classes: ', n_classes) model = LogReg( 'logreg', n_features, n_classes) #IMDB_NN_Model('imdb', n_features, H_0, H_1, n_classes) criterion = nn.CrossEntropyLoss() optimizer = torch.optim.Adam(model.parameters(), lr=0.001) size_batch = 256 loss_hist = [] loss_val_hist = [] acc_train = [] acc_val = [] # Train start = time.time() for t in range(n_iter): X_mini, y_mini = get_mini_batching(X_train, y_train, size_batch)