def process_dataset_appnp(epsilon): k = 20 alpha = 0.2 dataset = PubmedGraphDataset() graph = dataset[0] feat = graph.ndata.pop('feat') label = graph.ndata.pop('label') train_mask = graph.ndata.pop('train_mask') val_mask = graph.ndata.pop('val_mask') test_mask = graph.ndata.pop('test_mask') train_idx = th.nonzero(train_mask, as_tuple=False).squeeze() val_idx = th.nonzero(val_mask, as_tuple=False).squeeze() test_idx = th.nonzero(test_mask, as_tuple=False).squeeze() appnp = APPNPConv(k, alpha) id = th.eye(graph.number_of_nodes()).float() diff_adj = appnp(graph.add_self_loop(), id).numpy() diff_adj[diff_adj < epsilon] = 0 scaler = MinMaxScaler() scaler.fit(diff_adj) diff_adj = scaler.transform(diff_adj) diff_edges = np.nonzero(diff_adj) diff_weight = diff_adj[diff_edges] diff_graph = dgl.graph(diff_edges) return graph, diff_graph, feat, label, train_idx, val_idx, test_idx, diff_weight
def load_pubmed(): from dgl.data import PubmedGraphDataset data = PubmedGraphDataset() g = data[0] g.ndata['features'] = g.ndata['feat'] g.ndata['labels'] = g.ndata['label'] return g, data.num_labels
def load_data_default(dataset_name): if dataset_name in ['cora', 'citeseer', 'pubmed']: if dataset_name == 'cora': dataset = CoraGraphDataset() if dataset_name == 'citeseer': dataset = CiteseerGraphDataset() if dataset_name == 'pubmed': dataset = PubmedGraphDataset() graph = dataset[0] graph = graph.remove_self_loop().add_self_loop() print(graph) features = graph.ndata['feat'] labels = graph.ndata['label'] train_mask = graph.ndata['train_mask'] val_mask = graph.ndata['val_mask'] test_mask = graph.ndata['test_mask'] num_feats = features.shape[1] num_classes = int(labels.max().item() + 1) else: dataset = DglNodePropPredDataset(name=dataset_name) splitted_mask = dataset.get_idx_split() train_mask, val_mask, test_mask = splitted_mask[ 'train'], splitted_mask['valid'], splitted_mask['test'] graph, labels = dataset[0] features = graph.ndata["feat"] num_feats = features.shape[1] num_classes = (labels.max() + 1).item() # add reverse edges srcs, dsts = graph.all_edges() graph.add_edges(dsts, srcs) #add self-loop graph = graph.remove_self_loop().add_self_loop() return graph, features, labels, train_mask, val_mask, test_mask, num_feats, num_classes
def get_dataset(opt): # load and preprocess dataset if opt['dataset'] == 'cora': data = CoraGraphDataset() elif opt['dataset'] == 'citeseer': data = CiteseerGraphDataset() elif opt['dataset'] == 'pubmed': data = PubmedGraphDataset() else: raise ValueError('Unknown dataset: {}'.format(opt['dataset'])) return data
def load(name): if name == 'cora': dataset = CoraGraphDataset() elif name == 'citeseer': dataset = CiteseerGraphDataset() elif name == 'pubmed': dataset = PubmedGraphDataset() graph = dataset[0] train_mask = graph.ndata.pop('train_mask') test_mask = graph.ndata.pop('test_mask') feat = graph.ndata.pop('feat') labels = graph.ndata.pop('label') return graph, feat, labels, train_mask, test_mask
def data_load(args): # load and preprocess dataset if args.dataset == 'cora': data = CoraGraphDataset() elif args.dataset == 'citeseer': data = CiteseerGraphDataset() elif args.dataset == 'pubmed': data = PubmedGraphDataset() else: raise ValueError('Unknown dataset: {}'.format(args.dataset)) g = data[0] args.device = torch.device( args.device if torch.cuda.is_available() else 'cpu') g.int().to(args.device) args.in_feats = g.ndata['feat'].shape[1] args.num_classes = data.num_classes return g
def load_data(dataset): if dataset == 'cora': data = CoraGraphDataset() elif dataset == 'citeseer': data = CiteseerGraphDataset() elif dataset == 'pubmed': data = PubmedGraphDataset() g = data[0] features = g.ndata['feat'] labels = g.ndata['label'] train_mask = g.ndata['train_mask'] val_mask = g.ndata['val_mask'] test_mask = g.ndata['test_mask'] nxg = g.to_networkx() adj = nx.to_scipy_sparse_matrix(nxg, dtype=np.float) adj = preprocess_adj(adj) adj = sparse_mx_to_torch_sparse_tensor(adj) return adj, features, labels, train_mask, val_mask, test_mask
def main(): # check cuda device = f'cuda:{args.gpu}' if torch.cuda.is_available( ) and args.gpu >= 0 else 'cpu' # load data if args.dataset == 'Cora': dataset = CoraGraphDataset() elif args.dataset == 'Citeseer': dataset = CiteseerGraphDataset() elif args.dataset == 'Pubmed': dataset = PubmedGraphDataset() else: raise ValueError('Dataset {} is invalid.'.format(args.dataset)) g = dataset[0] g = dgl.add_self_loop(g) labels = g.ndata.pop('label').to(device).long() # load masks for train / test, valid is not used. train_mask = g.ndata.pop('train_mask') test_mask = g.ndata.pop('test_mask') train_idx = torch.nonzero(train_mask, as_tuple=False).squeeze().to(device) test_idx = torch.nonzero(test_mask, as_tuple=False).squeeze().to(device) g = g.to(device) # label propagation lp = LabelPropagation(args.num_layers, args.alpha) logits = lp(g, labels, mask=train_idx) test_acc = torch.sum(logits[test_idx].argmax( dim=1) == labels[test_idx]).item() / len(test_idx) print("Test Acc {:.4f}".format(test_acc))
import argparse, time import numpy as np import networkx as nx import torch import torch.nn as nn import torch.nn.functional as F import dgl from dgl.data import register_data_args from dgl.data import CoraGraphDataset, CiteseerGraphDataset, PubmedGraphDataset from gin import GIN_DGL t0 = time.time() data = PubmedGraphDataset(force_reload=True, verbose=False) g = data[0] print("data load time: {:.4f}\n".format(time.time() - t0)) device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') print("cuda: \n", torch.cuda.is_available()) g = g.int().to(device) features = g.ndata['feat'] labels = g.ndata['label'] train_mask = g.ndata['train_mask'] val_mask = g.ndata['val_mask'] test_mask = g.ndata['test_mask'] in_feats = features.shape[1] n_classes = data.num_labels n_edges = data.graph.number_of_edges()
def main(args): # Step 1: Prepare graph data and retrieve train/validation/test index ============================= # # Load from DGL dataset if args.dataset == 'Cora': dataset = CoraGraphDataset() elif args.dataset == 'Citeseer': dataset = CiteseerGraphDataset() elif args.dataset == 'Pubmed': dataset = PubmedGraphDataset() else: raise ValueError('Dataset {} is invalid.'.format(args.dataset)) graph = dataset[0] graph = dgl.add_self_loop(graph) # check cuda if args.gpu >= 0 and torch.cuda.is_available(): device = 'cuda:{}'.format(args.gpu) else: device = 'cpu' # retrieve the number of classes n_classes = dataset.num_classes # retrieve labels of ground truth labels = graph.ndata.pop('label').to(device).long() # Extract node features feats = graph.ndata.pop('feat').to(device) n_features = feats.shape[-1] # retrieve masks for train/validation/test train_mask = graph.ndata.pop('train_mask') val_mask = graph.ndata.pop('val_mask') test_mask = graph.ndata.pop('test_mask') train_idx = torch.nonzero(train_mask, as_tuple=False).squeeze().to(device) val_idx = torch.nonzero(val_mask, as_tuple=False).squeeze().to(device) test_idx = torch.nonzero(test_mask, as_tuple=False).squeeze().to(device) graph = graph.to(device) # Step 2: Create model =================================================================== # model = MixHop(in_dim=n_features, hid_dim=args.hid_dim, out_dim=n_classes, num_layers=args.num_layers, p=args.p, input_dropout=args.input_dropout, layer_dropout=args.layer_dropout, activation=torch.tanh, batchnorm=True) model = model.to(device) # Step 3: Create training components ===================================================== # loss_fn = nn.CrossEntropyLoss() opt = optim.SGD(model.parameters(), lr=args.lr, weight_decay=args.lamb) scheduler = optim.lr_scheduler.StepLR(opt, args.step_size, gamma=args.gamma) # Step 4: training epoches =============================================================== # acc = 0 no_improvement = 0 epochs = trange(args.epochs, desc='Accuracy & Loss') for _ in epochs: # Training using a full graph model.train() logits = model(graph, feats) # compute loss train_loss = loss_fn(logits[train_idx], labels[train_idx]) train_acc = torch.sum(logits[train_idx].argmax( dim=1) == labels[train_idx]).item() / len(train_idx) # backward opt.zero_grad() train_loss.backward() opt.step() # Validation using a full graph model.eval() with torch.no_grad(): valid_loss = loss_fn(logits[val_idx], labels[val_idx]) valid_acc = torch.sum(logits[val_idx].argmax( dim=1) == labels[val_idx]).item() / len(val_idx) # Print out performance epochs.set_description( 'Train Acc {:.4f} | Train Loss {:.4f} | Val Acc {:.4f} | Val loss {:.4f}' .format(train_acc, train_loss.item(), valid_acc, valid_loss.item())) if valid_acc < acc: no_improvement += 1 if no_improvement == args.early_stopping: print('Early stop.') break else: no_improvement = 0 acc = valid_acc scheduler.step() model.eval() logits = model(graph, feats) test_acc = torch.sum(logits[test_idx].argmax( dim=1) == labels[test_idx]).item() / len(test_idx) print("Test Acc {:.4f}".format(test_acc)) return test_acc
def main(args): # load and preprocess dataset if args.dataset == 'cora': data = CoraGraphDataset() elif args.dataset == 'citeseer': data = CiteseerGraphDataset() elif args.dataset == 'pubmed': data = PubmedGraphDataset() else: raise ValueError('Unknown dataset: {}'.format(args.dataset)) g = data[0] if args.gpu < 0: cuda = False ctx = mx.cpu(0) else: cuda = True ctx = mx.gpu(args.gpu) g = g.to(ctx) features = g.ndata['feat'] labels = mx.nd.array(g.ndata['label'], dtype="float32", ctx=ctx) train_mask = g.ndata['train_mask'] val_mask = g.ndata['val_mask'] test_mask = g.ndata['test_mask'] in_feats = features.shape[1] n_classes = data.num_labels n_edges = data.graph.number_of_edges() print("""----Data statistics------' #Edges %d #Classes %d #Train samples %d #Val samples %d #Test samples %d""" % (n_edges, n_classes, train_mask.sum().asscalar(), val_mask.sum().asscalar(), test_mask.sum().asscalar())) # add self loop g = dgl.remove_self_loop(g) g = dgl.add_self_loop(g) n_edges = g.number_of_edges() us, vs = g.edges() us = us.asnumpy() vs = vs.asnumpy() pseudo = [] for i in range(g.number_of_edges()): pseudo.append([ 1 / np.sqrt(g.in_degree(us[i])), 1 / np.sqrt(g.in_degree(vs[i])) ]) pseudo = nd.array(pseudo, ctx=ctx) # create GraphSAGE model model = MoNet(g, in_feats, args.n_hidden, n_classes, args.n_layers, args.pseudo_dim, args.n_kernels, args.dropout ) model.initialize(ctx=ctx) n_train_samples = train_mask.sum().asscalar() loss_fcn = gluon.loss.SoftmaxCELoss() print(model.collect_params()) trainer = gluon.Trainer(model.collect_params(), 'adam', {'learning_rate': args.lr, 'wd': args.weight_decay}) # initialize graph dur = [] for epoch in range(args.n_epochs): if epoch >= 3: t0 = time.time() # forward with mx.autograd.record(): pred = model(features, pseudo) loss = loss_fcn(pred, labels, mx.nd.expand_dims(train_mask, 1)) loss = loss.sum() / n_train_samples loss.backward() trainer.step(batch_size=1) if epoch >= 3: loss.asscalar() dur.append(time.time() - t0) acc = evaluate(model, features, pseudo, labels, val_mask) print("Epoch {:05d} | Time(s) {:.4f} | Loss {:.4f} | Accuracy {:.4f} | " "ETputs(KTEPS) {:.2f}". format( epoch, np.mean(dur), loss.asscalar(), acc, n_edges / np.mean(dur) / 1000)) # test set accuracy acc = evaluate(model, features, pseudo, labels, test_mask) print("Test accuracy {:.2%}".format(acc))
SGC implementation in DGL. """ import argparse import textwrap import tensorflow as tf import tensorflow_addons as tfa from dgl.data import CiteseerGraphDataset, CoraGraphDataset, PubmedGraphDataset from dgl.nn.tensorflow.conv import SGConv _DATASETS = { 'citeseer': CiteseerGraphDataset(verbose=False), 'cora': CoraGraphDataset(verbose=False), 'pubmed': PubmedGraphDataset(verbose=False) } def load_data(dataset): return _DATASETS[dataset] def _sum_boolean_tensor(x): return tf.reduce_sum(tf.cast(x, dtype='int64')) def describe_data(data): g = data[0] n_edges = g.number_of_edges()
def load_pubmed(): from dgl.data import PubmedGraphDataset data = PubmedGraphDataset() g = data[0] g.ndata['labels'] = g.ndata.pop('label') return g
def main(args): # load and preprocess dataset if args.dataset == 'cora': data = CoraGraphDataset() elif args.dataset == 'citeseer': data = CiteseerGraphDataset() elif args.dataset == 'pubmed': data = PubmedGraphDataset() else: raise ValueError('Unknown dataset: {}'.format(args.dataset)) g = data[0] if args.gpu < 0: device = "/cpu:0" else: device = "/gpu:{}".format(args.gpu) g = g.to(device) with tf.device(device): features = g.ndata['feat'] labels = g.ndata['label'] train_mask = g.ndata['train_mask'] val_mask = g.ndata['val_mask'] test_mask = g.ndata['test_mask'] in_feats = features.shape[1] n_classes = data.num_labels n_edges = data.graph.number_of_edges() # add self loop if args.self_loop: g = dgl.remove_self_loop(g) g = dgl.add_self_loop(g) n_edges = g.number_of_edges() # create DGI model dgi = DGI( g, in_feats, args.n_hidden, args.n_layers, tf.keras.layers.PReLU( alpha_initializer=tf.constant_initializer(0.25)), args.dropout) dgi_optimizer = tf.keras.optimizers.Adam(learning_rate=args.dgi_lr) # train deep graph infomax cnt_wait = 0 best = 1e9 best_t = 0 dur = [] for epoch in range(args.n_dgi_epochs): if epoch >= 3: t0 = time.time() with tf.GradientTape() as tape: loss = dgi(features) # Manually Weight Decay # We found Tensorflow has a different implementation on weight decay # of Adam(W) optimizer with PyTorch. And this results in worse results. # Manually adding weights to the loss to do weight decay solves this problem. for weight in dgi.trainable_weights: loss = loss + \ args.weight_decay * tf.nn.l2_loss(weight) grads = tape.gradient(loss, dgi.trainable_weights) dgi_optimizer.apply_gradients(zip(grads, dgi.trainable_weights)) if loss < best: best = loss best_t = epoch cnt_wait = 0 dgi.save_weights('best_dgi.pkl') else: cnt_wait += 1 if cnt_wait == args.patience: print('Early stopping!') break if epoch >= 3: dur.append(time.time() - t0) print("Epoch {:05d} | Time(s) {:.4f} | Loss {:.4f} | " "ETputs(KTEPS) {:.2f}".format(epoch, np.mean(dur), loss.numpy().item(), n_edges / np.mean(dur) / 1000)) # create classifier model classifier = Classifier(args.n_hidden, n_classes) classifier_optimizer = tf.keras.optimizers.Adam( learning_rate=args.classifier_lr) # train classifier print('Loading {}th epoch'.format(best_t)) dgi.load_weights('best_dgi.pkl') embeds = dgi.encoder(features, corrupt=False) embeds = tf.stop_gradient(embeds) dur = [] loss_fcn = tf.keras.losses.SparseCategoricalCrossentropy( from_logits=True) for epoch in range(args.n_classifier_epochs): if epoch >= 3: t0 = time.time() with tf.GradientTape() as tape: preds = classifier(embeds) loss = loss_fcn(labels[train_mask], preds[train_mask]) # Manually Weight Decay # We found Tensorflow has a different implementation on weight decay # of Adam(W) optimizer with PyTorch. And this results in worse results. # Manually adding weights to the loss to do weight decay solves this problem. # In original code, there's no weight decay applied in this part # link: https://github.com/PetarV-/DGI/blob/master/execute.py#L121 # for weight in classifier.trainable_weights: # loss = loss + \ # args.weight_decay * tf.nn.l2_loss(weight) grads = tape.gradient(loss, classifier.trainable_weights) classifier_optimizer.apply_gradients( zip(grads, classifier.trainable_weights)) if epoch >= 3: dur.append(time.time() - t0) acc = evaluate(classifier, embeds, labels, val_mask) print( "Epoch {:05d} | Time(s) {:.4f} | Loss {:.4f} | Accuracy {:.4f} | " "ETputs(KTEPS) {:.2f}".format(epoch, np.mean(dur), loss.numpy().item(), acc, n_edges / np.mean(dur) / 1000)) print() acc = evaluate(classifier, embeds, labels, test_mask) print("Test Accuracy {:.4f}".format(acc))
def train(procid,args): # load and preprocess dataset assert procid >= 0 os.environ['MASTER_ADDR'] = '127.0.0.1' os.environ['MASTER_PORT'] = '12345' if args.dataset == 'cora': data = CoraGraphDataset() elif args.dataset == 'citeseer': data = CiteseerGraphDataset() elif args.dataset == 'pubmed': data = PubmedGraphDataset() elif args.dataset == 'reddit': data = RedditDataset() else: raise ValueError('Unknown dataset: {}'.format(args.dataset)) g = data[0] #data = args.data #g = args.data[0] #g.create_formats_() print("New Proc! ",procid) #return g device = torch.device(args.devices_name_list[procid]) dist_init_method = 'tcp://{master_ip}:{master_port}'.format( master_ip='127.0.0.1', master_port='12345') world_size = args.ngpus torch.distributed.init_process_group(backend="nccl", init_method=dist_init_method, world_size = world_size, rank = procid) #torch.cuda.set_device(device) #st = pg.Storage(g,[device],[args.PV_list[procid]],[args.TV_list[procid]]) # use pagraph st = pg.Storage(g=g,data=g.ndata,cache_rate=args.cache_rate, nodes=args.PV_list[procid],gpu=args.devices_name_list[procid],cpu='cpu') if(True): features = g.ndata.pop('feat') labels = g.ndata.pop('label') train_mask = g.ndata.pop('train_mask') val_mask = g.ndata.pop('val_mask') test_mask = g.ndata.pop('test_mask') in_feats = features.shape[1] n_classes = data.num_labels n_edges = data.graph.number_of_edges() print("""----Data statistics------' #Edges %d #Classes %d #Train samples %d #Val samples %d #Test samples %d""" % (n_edges, n_classes, train_mask.int().sum().item(), val_mask.int().sum().item(), test_mask.int().sum().item())) del features #release memory # add self loop ''' if args.self_loop: g = dgl.remove_self_loop(g) g = dgl.add_self_loop(g) ''' # create GCN model model = MyGCN( in_feats, args.n_hidden, n_classes, args.n_layers, F.relu, args.dropout, ) model = model.to(device) model = DistributedDataParallel(model, device_ids = [device], output_device = device) # set sampler fanouts=[] for i in range(args.n_layers + 1): fanouts.append(args.neighbor_number) ''' example: fanout=[2,2,2,2] or [3,3,3] ... ''' sampler = dgl.dataloading.MultiLayerNeighborSampler(fanouts) train_nids = args.TV_list[procid] dataloader = dgl.dataloading.NodeDataLoader( g, train_nids, sampler, batch_size=args.batch_size, shuffle=True, drop_last=True, num_workers=0) # set loss function loss_fcn = torch.nn.CrossEntropyLoss() # use optimizer optimizer = torch.optim.Adam(model.parameters(), lr = args.lr) # initialize graph dur = [] # Sync if(args.ngpus > 1): torch.distributed.barrier() #Start trainning model.train() for epoch in range(args.n_epochs): # time record #if epoch >= 3: tS=[0.0,0.0,0.0,0.0,0.0,0.0] t0 = time.time() # forward #Loss=torch.tensor([0.0],device=device,required_grad=False) for count,(in_nodes,out_nodes,blocks) in enumerate(dataloader): t1=time.time() blocks=[b.to(device) for b in blocks] t2=time.time() feat_in = st.Query(0,in_nodes,'feat') labels_out = st.Query(0,out_nodes,'label') t3=time.time() # forward feat_out = model(blocks,feat_in) t4=time.time() loss = loss_fcn(feat_out,labels_out) #Loss=Loss+loss.detach() t5=time.time() optimizer.zero_grad() loss.backward() optimizer.step() t6=time.time() tS[1]=tS[1]+t2-t1 tS[2]=tS[2]+t3-t2 tS[3]=tS[3]+t4-t3 tS[4]=tS[4]+t5-t4 tS[5]=tS[5]+t6-t5 tE=time.time() #logits = model(features) #loss = loss_fcn(logits[train_mask], labels[train_mask]) #optimizer.zero_grad() #loss.backward() #optimizer.step() #if epoch >= 3: dur.append(time.time() - t0) acc = 0.0 #evaluate(model, features, labels, val_mask) print("Epoch {:05d} | Time(s) {:.4f} | Loss {:.4f} | Accuracy {:.4f} | " "ETputs(KTEPS) {:.2f}". format(epoch, np.mean(dur), loss.item(), acc, n_edges / np.mean(dur) / 1000)) #for i in range(1,6): print(tS[1:],'\nTotal:',tE-t0," s ") #Finish trainning # Sync if(args.ngpus > 1): torch.distributed.barrier() model.eval() print("____________________________")
return loss if __name__ == '__main__': # Step 1: Prepare graph data and retrieve train/validation/test index ============================= # # Load from DGL dataset args = argument() print(args) if args.dataname == 'cora': dataset = CoraGraphDataset() elif args.dataname == 'citeseer': dataset = CiteseerGraphDataset() elif args.dataname == 'pubmed': dataset = PubmedGraphDataset() graph = dataset[0] graph = dgl.add_self_loop(graph) device = args.device # retrieve the number of classes n_classes = dataset.num_classes # retrieve labels of ground truth labels = graph.ndata.pop('label').to(device).long() # Extract node features feats = graph.ndata.pop('feat').to(device) n_features = feats.shape[-1]
def main(args): # Step 1: Prepare graph data and retrieve train/validation/test index ============================= # # Load from DGL dataset if args.dataset == 'Cora': dataset = CoraGraphDataset() elif args.dataset == 'Citeseer': dataset = CiteseerGraphDataset() elif args.dataset == 'Pubmed': dataset = PubmedGraphDataset() else: raise ValueError('Dataset {} is invalid.'.format(args.dataset)) graph = dataset[0] graph = graph.remove_self_loop().add_self_loop() # check cuda if args.gpu >= 0 and torch.cuda.is_available(): device = 'cuda:{}'.format(args.gpu) else: device = 'cpu' # retrieve the number of classes n_classes = dataset.num_classes # retrieve labels of ground truth labels = graph.ndata.pop('label').to(device).long() # Extract node features feats = graph.ndata.pop('feat').to(device) n_features = feats.shape[-1] # retrieve masks for train/validation/test train_mask = graph.ndata.pop('train_mask') val_mask = graph.ndata.pop('val_mask') test_mask = graph.ndata.pop('test_mask') train_idx = torch.nonzero(train_mask, as_tuple=False).squeeze().to(device) val_idx = torch.nonzero(val_mask, as_tuple=False).squeeze().to(device) test_idx = torch.nonzero(test_mask, as_tuple=False).squeeze().to(device) graph = graph.to(device) # Step 2: Create model =================================================================== # model = DAGNN(k=args.k, in_dim=n_features, hid_dim=args.hid_dim, out_dim=n_classes, batchnorm=args.batchnorm, dropout=args.dropout, drop_bef=args.drop_bef) model = model.to(device) # Step 3: Create training components ===================================================== # loss_fn = F.cross_entropy opt = torch.optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.lamb) # Step 4: training epoches =============================================================== # loss = None best_acc = None no_improvement = 0 epochs = trange(args.epochs, desc='Accuracy & Loss') for _ in epochs: model.train() logits = model(graph, feats) # compute loss train_loss = loss_fn(logits[train_idx], labels[train_idx]) # backward opt.zero_grad() train_loss.backward() opt.step() # Validation using a full graph train_loss, train_acc, valid_loss, valid_acc, test_loss, test_acc = evaluate( model, graph, feats, labels, (train_idx, val_idx, test_idx)) # Print out performance epochs.set_description( 'Train Acc {:.4f} | Train Loss {:.4f} | Val Acc {:.4f} | Val loss {:.4f}' .format(train_acc, train_loss.item(), valid_acc, valid_loss.item())) if loss is None: loss = valid_loss best_acc = test_acc else: if valid_loss > loss: no_improvement += 1 if no_improvement == args.early_stopping: print('Early stop.') break else: no_improvement = 0 loss = valid_loss best_acc = test_acc print("Test Acc {:.4f}".format(best_acc)) return best_acc
def main(args): # load and preprocess dataset if args.dataset == 'cora': data = CoraGraphDataset() elif args.dataset == 'citeseer': data = CiteseerGraphDataset() elif args.dataset == 'pubmed': data = PubmedGraphDataset() elif args.dataset == 'reddit': data = RedditDataset() else: raise ValueError('Unknown dataset: {}'.format(args.dataset)) g = data[0] g.create_formats_() st = pg.Storage(g=g, data=g.ndata, gpu='cuda:0', cache_rate=args.cache_rate, nodes=g.nodes()) if (True): features = g.ndata.pop('feat') labels = g.ndata.pop('label') train_mask = g.ndata.pop('train_mask') val_mask = g.ndata.pop('val_mask') test_mask = g.ndata.pop('test_mask') in_feats = features.shape[1] n_classes = data.num_labels n_edges = data.graph.number_of_edges() #return g device = gpu = 'cuda:0' cpu = 'cpu' print("""----Data statistics------' #Edges %d #Classes %d #Train samples %d #Val samples %d #Test samples %d""" % (n_edges, n_classes, train_mask.int().sum().item(), val_mask.int().sum().item(), test_mask.int().sum().item())) # create GCN model model = MyGCN( in_feats, args.n_hidden, n_classes, args.n_layers, F.relu, args.dropout, ) model = model.to(gpu) # set sampler fanouts = [] for i in range(args.n_layers + 1): fanouts.append(args.neighbor_number) ''' example: fanout=[2,2,2,2] or [3,3,3] ... ''' sampler = dgl.dataloading.MultiLayerNeighborSampler(fanouts) train_nids = torch.arange(0, len(train_mask), 1, dtype=torch.long) dataloader = dgl.dataloading.NodeDataLoader(g, train_nids, sampler, batch_size=args.batch_size, shuffle=True, drop_last=True, num_workers=0) # set loss function loss_fcn = torch.nn.CrossEntropyLoss() # use optimizer optimizer = torch.optim.Adam(model.parameters(), lr=args.lr) # initialize graph dur = [] #Start trainning model.train() for epoch in range(args.n_epochs): # time record #if epoch >= 3: tS = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0] t0 = time.time() # forward for count, (in_nodes, out_nodes, blocks) in enumerate(dataloader): t1 = time.time() blocks = [b.to(device) for b in blocks] t2 = time.time() feat_in = st.Query(fname='feat', nodes=in_nodes) labels_out = st.Query(fname='label', nodes=out_nodes) t3 = time.time() # forward feat_out = model(blocks, feat_in) t4 = time.time() loss = loss_fcn(feat_out, labels_out) #Loss=Loss+loss.detach() t5 = time.time() optimizer.zero_grad() loss.backward() optimizer.step() t6 = time.time() tS[1] = tS[1] + t2 - t1 tS[2] = tS[2] + t3 - t2 tS[3] = tS[3] + t4 - t3 tS[4] = tS[4] + t5 - t4 tS[5] = tS[5] + t6 - t5 tE = time.time() #logits = model(features) #loss = loss_fcn(logits[train_mask], labels[train_mask]) #optimizer.zero_grad() #loss.backward() #optimizer.step() #if epoch >= 3: dur.append(time.time() - t0) acc = 0.0 #evaluate(model, features, labels, val_mask) print( "Epoch {:05d} | Time(s) {:.4f} | Loss {:.4f} | Accuracy {:.4f} | " "ETputs(KTEPS) {:.2f}".format(epoch, np.mean(dur), loss.item(), acc, n_edges / np.mean(dur) / 1000)) #for i in range(1,6): print(tS[1:], '\nTotal:', tE - t0, " s ") #Finish trainning model.eval() print("____________________________")
def main(args): # load and preprocess dataset if args.dataset == 'cora': data = CoraGraphDataset() elif args.dataset == 'citeseer': data = CiteseerGraphDataset() elif args.dataset == 'pubmed': data = PubmedGraphDataset() else: raise ValueError('Unknown dataset: {}'.format(args.dataset)) g = data[0] if args.gpu < 0: cuda = False ctx = mx.cpu(0) else: cuda = True ctx = mx.gpu(args.gpu) g = g.to(ctx) features = g.ndata['feat'] labels = mx.nd.array(g.ndata['label'], dtype="float32", ctx=ctx) train_mask = g.ndata['train_mask'] val_mask = g.ndata['val_mask'] test_mask = g.ndata['test_mask'] in_feats = features.shape[1] n_classes = data.num_labels n_edges = data.graph.number_of_edges() print("""----Data statistics------' #Edges %d #Classes %d #Train samples %d #Val samples %d #Test samples %d""" % (n_edges, n_classes, train_mask.sum().asscalar(), val_mask.sum().asscalar(), test_mask.sum().asscalar())) # add self loop if args.self_loop: g = dgl.remove_self_loop(g) g = dgl.add_self_loop(g) # normalization in_degs = g.in_degrees().astype('float32') out_degs = g.out_degrees().astype('float32') in_norm = mx.nd.power(in_degs, -0.5) out_norm = mx.nd.power(out_degs, -0.5) if cuda: in_norm = in_norm.as_in_context(ctx) out_norm = out_norm.as_in_context(ctx) g.ndata['in_norm'] = mx.nd.expand_dims(in_norm, 1) g.ndata['out_norm'] = mx.nd.expand_dims(out_norm, 1) model = GCN( g, args.n_hidden, n_classes, args.n_layers, 'relu', args.dropout, ) model.initialize(ctx=ctx) n_train_samples = train_mask.sum().asscalar() loss_fcn = gluon.loss.SoftmaxCELoss() # use optimizer print(model.collect_params()) trainer = gluon.Trainer(model.collect_params(), 'adam', { 'learning_rate': args.lr, 'wd': args.weight_decay }) # initialize graph dur = [] for epoch in range(args.n_epochs): if epoch >= 3: t0 = time.time() # forward with mx.autograd.record(): pred = model(features) loss = loss_fcn(pred, labels, mx.nd.expand_dims(train_mask, 1)) loss = loss.sum() / n_train_samples loss.backward() trainer.step(batch_size=1) if epoch >= 3: dur.append(time.time() - t0) acc = evaluate(model, features, labels, val_mask) print( "Epoch {:05d} | Time(s) {:.4f} | Loss {:.4f} | Accuracy {:.4f} | " "ETputs(KTEPS) {:.2f}".format(epoch, np.mean(dur), loss.asscalar(), acc, n_edges / np.mean(dur) / 1000)) # test set accuracy acc = evaluate(model, features, labels, test_mask) print("Test accuracy {:.2%}".format(acc))
def main(args): # load and preprocess dataset if args.dataset == 'cora': data = CoraGraphDataset() elif args.dataset == 'citeseer': data = CiteseerGraphDataset() elif args.dataset == 'pubmed': data = PubmedGraphDataset() else: raise ValueError('Unknown dataset: {}'.format(args.dataset)) g = data[0] if args.gpu < 0: cuda = False else: cuda = True g = g.int().to(args.gpu) features = g.ndata['feat'] labels = g.ndata['label'] train_mask = g.ndata['train_mask'] val_mask = g.ndata['val_mask'] test_mask = g.ndata['test_mask'] in_feats = features.shape[1] n_classes = data.num_labels n_edges = g.number_of_edges() print("""----Data statistics------' #Edges %d #Classes %d #Train samples %d #Val samples %d #Test samples %d""" % (n_edges, n_classes, train_mask.int().sum().item(), val_mask.int().sum().item(), test_mask.int().sum().item())) n_edges = g.number_of_edges() # add self loop g = dgl.remove_self_loop(g) g = dgl.add_self_loop(g) # create SGC model model = SGConv(in_feats, n_classes, k=2, cached=True, bias=args.bias) if cuda: model.cuda() loss_fcn = torch.nn.CrossEntropyLoss() # use optimizer optimizer = torch.optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.weight_decay) # initialize graph dur = [] for epoch in range(args.n_epochs): model.train() if epoch >= 3: t0 = time.time() # forward logits = model(g, features) # only compute the train set loss = loss_fcn(logits[train_mask], labels[train_mask]) optimizer.zero_grad() loss.backward() optimizer.step() if epoch >= 3: dur.append(time.time() - t0) acc = evaluate(model, g, features, labels, val_mask) print("Epoch {:05d} | Time(s) {:.4f} | Loss {:.4f} | Accuracy {:.4f} | " "ETputs(KTEPS) {:.2f}". format(epoch, np.mean(dur), loss.item(), acc, n_edges / np.mean(dur) / 1000)) print() acc = evaluate(model, g, features, labels, test_mask) print("Test Accuracy {:.4f}".format(acc))
def main(args): # Step 1: Prepare graph data and retrieve train/validation/test index ============================= # # Load dataset dataset = PubmedGraphDataset() graph = dataset[0] # check cuda if args.gpu >= 0 and th.cuda.is_available(): device = 'cuda:{}'.format(args.gpu) else: device = 'cpu' num_classes = dataset.num_classes # retrieve label of ground truth label = graph.ndata['label'].to(device) # Extract node features feat = graph.ndata['feat'].to(device) # retrieve masks for train/validation/test train_mask = graph.ndata['train_mask'] val_mask = graph.ndata['val_mask'] test_mask = graph.ndata['test_mask'] train_idx = th.nonzero(train_mask, as_tuple=False).squeeze(1).to(device) val_idx = th.nonzero(val_mask, as_tuple=False).squeeze(1).to(device) test_idx = th.nonzero(test_mask, as_tuple=False).squeeze(1).to(device) graph = graph.to(device) # Step 2: Create model =================================================================== # if args.lazy: model = GeniePathLazy(in_dim=feat.shape[-1], out_dim=num_classes, hid_dim=args.hid_dim, num_layers=args.num_layers, num_heads=args.num_heads, residual=args.residual) else: model = GeniePath(in_dim=feat.shape[-1], out_dim=num_classes, hid_dim=args.hid_dim, num_layers=args.num_layers, num_heads=args.num_heads, residual=args.residual) model = model.to(device) # Step 3: Create training components ===================================================== # loss_fn = th.nn.CrossEntropyLoss() optimizer = optim.Adam(model.parameters(), lr=args.lr) # Step 4: training epochs =============================================================== # for epoch in range(args.max_epoch): # Training and validation model.train() logits = model(graph, feat) # compute loss tr_loss = loss_fn(logits[train_idx], label[train_idx]) tr_acc = accuracy_score(label[train_idx].cpu(), logits[train_idx].argmax(dim=1).cpu()) # validation valid_loss = loss_fn(logits[val_idx], label[val_idx]) valid_acc = accuracy_score(label[val_idx].cpu(), logits[val_idx].argmax(dim=1).cpu()) # backward optimizer.zero_grad() tr_loss.backward() optimizer.step() # Print out performance print( "In epoch {}, Train ACC: {:.4f} | Train Loss: {:.4f}; Valid ACC: {:.4f} | Valid loss: {:.4f}" .format(epoch, tr_acc, tr_loss.item(), valid_acc, valid_loss.item())) # Test after all epoch model.eval() # forward logits = model(graph, feat) # compute loss test_loss = loss_fn(logits[test_idx], label[test_idx]) test_acc = accuracy_score(label[test_idx].cpu(), logits[test_idx].argmax(dim=1).cpu()) print("Test ACC: {:.4f} | Test loss: {:.4f}".format( test_acc, test_loss.item()))
def main(args): # load and preprocess dataset set_seeds(args.rand_seed) if args.dataset == 'cora': data = CoraGraphDataset() elif args.dataset == 'citeseer': data = CiteseerGraphDataset() elif args.dataset == 'pubmed': data = PubmedGraphDataset() else: raise ValueError('Unknown dataset: {}'.format(args.dataset)) g = data[0] if args.gpu < 0: cuda = False else: cuda = True g = g.int().to(args.gpu) features = g.ndata['feat'] labels = g.ndata['label'] train_mask = g.ndata['train_mask'] val_mask = g.ndata['val_mask'] test_mask = g.ndata['test_mask'] if args.shuffle: print('Random splitting...') #### train_dev_test_tuple = (train_mask.sum().data.item(), val_mask.sum().data.item(), test_mask.sum().data.item()) train_mask, val_mask, test_mask = random_split( N=g.num_nodes(), train_dev_test_tuple=train_dev_test_tuple, random_seed=args.rand_seed) #### else: print('standard splitting') num_feats = features.shape[1] n_classes = data.num_labels n_edges = data.graph.number_of_edges() print("""----Data statistics------' #Edges %d #Classes %d #Train samples %d #Val samples %d #Test samples %d""" % (n_edges, n_classes, train_mask.int().sum().item(), val_mask.int().sum().item(), test_mask.int().sum().item())) # add self loop g = dgl.remove_self_loop(g) g = dgl.add_self_loop(g) n_edges = g.number_of_edges() # create model heads = ([args.num_heads] * args.num_layers) + [args.num_out_heads] model = GATFF(g, args.num_layers, num_feats, args.num_hidden, n_classes, heads, F.elu, args.in_drop, args.attn_drop, args.negative_slope, args.residual) print(model) if args.early_stop: stopper = EarlyStopping(patience=100) if cuda: model.cuda() loss_fcn = torch.nn.CrossEntropyLoss() # use optimizer optimizer = torch.optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.weight_decay) # initialize graph dur = [] best_val_acc = 0 test_acc = 0 for epoch in range(args.epochs): model.train() if epoch >= 3: t0 = time.time() # forward logits = model(features) loss = loss_fcn(logits[train_mask], labels[train_mask]) optimizer.zero_grad() loss.backward() optimizer.step() if epoch >= 3: dur.append(time.time() - t0) train_acc = accuracy(logits[train_mask], labels[train_mask]) if args.fastmode: val_acc = accuracy(logits[val_mask], labels[val_mask]) else: val_acc = evaluate(model, features, labels, val_mask) if args.early_stop: if stopper.step(val_acc, model): break if best_val_acc < val_acc: best_val_acc = val_acc test_acc = evaluate(model, features, labels, test_mask) print("Epoch {:05d} | Time(s) {:.4f} | Loss {:.4f} | TrainAcc {:.4f} |" " ValAcc {:.4f} TestAcc {:.4f}| ETputs(KTEPS) {:.2f}".format( epoch, np.mean(dur), loss.item(), train_acc, val_acc, test_acc, n_edges / np.mean(dur) / 1000)) print() if args.early_stop: model.load_state_dict(torch.load('es_checkpoint.pt')) acc = evaluate(model, features, labels, test_mask) print("Test Accuracy {:.4f}".format(acc)) print('Test Accuracy {:.4f}'.format(test_acc)) return test_acc
# coding: utf-8 """ @author: Nelson Zhao @date: 2021/5/13 10:28 PM @email: [email protected] """ import torch import torch.nn.functional as F from dgl.data import PubmedGraphDataset from dgl import DGLGraph from .geniepath import GeniePath pub_data = PubmedGraphDataset() features = torch.FloatTensor(pub_data.features) labels = torch.LongTensor(pub_data.labels) train_mask = torch.BoolTensor(pub_data.train_mask) val_mask = torch.BoolTensor(pub_data.val_mask) test_mask = torch.BoolTensor(pub_data.test_mask) g = DGLGraph(pub_data.graph) g = g.add_self_loop() # hyper-parameters in_dim = features.size()[-1] h_dim = 8 out_dim = pub_data.num_classes depth = 3 lazy = True
def dgl_main(): # Load from DGL dataset if args.dataset == 'cora': dataset = CoraGraphDataset(reverse_edge=False) elif args.dataset == 'citeseer': dataset = CiteseerGraphDataset(reverse_edge=False) elif args.dataset == 'pubmed': dataset = PubmedGraphDataset(reverse_edge=False) else: raise NotImplementedError graph = dataset[0] # Extract node features feats = graph.ndata.pop('feat').to(device) in_dim = feats.shape[-1] # generate input adj_orig = graph.adjacency_matrix().to_dense() # build test set with 10% positive links train_edge_idx, val_edges, val_edges_false, test_edges, test_edges_false = mask_test_edges_dgl( graph, adj_orig) graph = graph.to(device) # create train graph train_edge_idx = torch.tensor(train_edge_idx).to(device) train_graph = dgl.edge_subgraph(graph, train_edge_idx, relabel_nodes=False) train_graph = train_graph.to(device) adj = train_graph.adjacency_matrix().to_dense().to(device) # compute loss parameters weight_tensor, norm = compute_loss_para(adj) # create model vgae_model = model.VGAEModel(in_dim, args.hidden1, args.hidden2) vgae_model = vgae_model.to(device) # create training component optimizer = torch.optim.Adam(vgae_model.parameters(), lr=args.learning_rate) print('Total Parameters:', sum([p.nelement() for p in vgae_model.parameters()])) # create training epoch for epoch in range(args.epochs): t = time.time() # Training and validation using a full graph vgae_model.train() logits = vgae_model.forward(graph, feats) # compute loss loss = norm * F.binary_cross_entropy( logits.view(-1), adj.view(-1), weight=weight_tensor) kl_divergence = 0.5 / logits.size(0) * ( 1 + 2 * vgae_model.log_std - vgae_model.mean**2 - torch.exp(vgae_model.log_std)**2).sum(1).mean() loss -= kl_divergence # backward optimizer.zero_grad() loss.backward() optimizer.step() train_acc = get_acc(logits, adj) val_roc, val_ap = get_scores(val_edges, val_edges_false, logits) # Print out performance print("Epoch:", '%04d' % (epoch + 1), "train_loss=", "{:.5f}".format(loss.item()), "train_acc=", "{:.5f}".format(train_acc), "val_roc=", "{:.5f}".format(val_roc), "val_ap=", "{:.5f}".format(val_ap), "time=", "{:.5f}".format(time.time() - t)) test_roc, test_ap = get_scores(test_edges, test_edges_false, logits) # roc_means.append(test_roc) # ap_means.append(test_ap) print("End of training!", "test_roc=", "{:.5f}".format(test_roc), "test_ap=", "{:.5f}".format(test_ap))
if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument("--dataset", type=str, default="cora", help="Dataset name ('cora', 'citeseer', 'pubmed').") args = parser.parse_args() print(f'Training with DGL built-in GraphConv module.') # load and preprocess dataset transform = AddSelfLoop() # by default, it will first remove self-loops to prevent duplication if args.dataset == 'cora': data = CoraGraphDataset(transform=transform) elif args.dataset == 'citeseer': data = CiteseerGraphDataset(transform=transform) elif args.dataset == 'pubmed': data = PubmedGraphDataset(transform=transform) else: raise ValueError('Unknown dataset: {}'.format(args.dataset)) g = data[0] device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') g = g.int().to(device) features = g.ndata['feat'] labels = g.ndata['label'] masks = g.ndata['train_mask'], g.ndata['val_mask'], g.ndata['test_mask'] # normalization degs = g.in_degrees().float() norm = torch.pow(degs, -0.5).to(device) norm[torch.isinf(norm)] = 0 g.ndata['norm'] = norm.unsqueeze(1)
def main(args): # load and preprocess dataset if args.dataset == 'cora': data = CoraGraphDataset() elif args.dataset == 'citeseer': data = CiteseerGraphDataset() elif args.dataset == 'pubmed': data = PubmedGraphDataset() else: raise ValueError('Unknown dataset: {}'.format(args.dataset)) g = data[0] if args.gpu < 0: cuda = False ctx = mx.cpu(0) else: cuda = True ctx = mx.gpu(args.gpu) g = g.to(ctx) features = g.ndata['feat'] labels = mx.nd.array(g.ndata['label'], dtype="float32", ctx=ctx) mask = g.ndata['train_mask'] mask = mx.nd.array(np.nonzero(mask.asnumpy())[0], ctx=ctx) val_mask = g.ndata['val_mask'] val_mask = mx.nd.array(np.nonzero(val_mask.asnumpy())[0], ctx=ctx) test_mask = g.ndata['test_mask'] test_mask = mx.nd.array(np.nonzero(test_mask.asnumpy())[0], ctx=ctx) in_feats = features.shape[1] n_classes = data.num_labels n_edges = data.graph.number_of_edges() g = dgl.remove_self_loop(g) g = dgl.add_self_loop(g) # create model heads = ([args.num_heads] * args.num_layers) + [args.num_out_heads] model = GAT(g, args.num_layers, in_feats, args.num_hidden, n_classes, heads, elu, args.in_drop, args.attn_drop, args.alpha, args.residual) if args.early_stop: stopper = EarlyStopping(patience=100) model.initialize(ctx=ctx) # use optimizer trainer = gluon.Trainer(model.collect_params(), 'adam', {'learning_rate': args.lr}) dur = [] for epoch in range(args.epochs): if epoch >= 3: t0 = time.time() # forward with mx.autograd.record(): logits = model(features) loss = mx.nd.softmax_cross_entropy(logits[mask].squeeze(), labels[mask].squeeze()) loss.backward() trainer.step(mask.shape[0]) if epoch >= 3: dur.append(time.time() - t0) print("Epoch {:05d} | Loss {:.4f} | Time(s) {:.4f} | ETputs(KTEPS) {:.2f}".format( epoch, loss.asnumpy()[0], np.mean(dur), n_edges / np.mean(dur) / 1000)) val_accuracy = evaluate(model, features, labels, val_mask) print("Validation Accuracy {:.4f}".format(val_accuracy)) if args.early_stop: if stopper.step(val_accuracy, model): break print() if args.early_stop: model.load_parameters('model.param') test_accuracy = evaluate(model, features, labels, test_mask) print("Test Accuracy {:.4f}".format(test_accuracy))
def main(args): # load and preprocess dataset if args.dataset == 'cora': data = CoraGraphDataset() elif args.dataset == 'citeseer': data = CiteseerGraphDataset() elif args.dataset == 'pubmed': data = PubmedGraphDataset() else: raise ValueError('Unknown dataset: {}'.format(args.dataset)) g = data[0] if args.gpu < 0: device = "/cpu:0" else: device = "/gpu:{}".format(args.gpu) g = g.to(device) with tf.device(device): features = g.ndata['feat'] labels = g.ndata['label'] train_mask = g.ndata['train_mask'] val_mask = g.ndata['val_mask'] test_mask = g.ndata['test_mask'] in_feats = features.shape[1] n_classes = data.num_labels n_edges = data.graph.number_of_edges() print("""----Data statistics------' #Edges %d #Classes %d #Train samples %d #Val samples %d #Test samples %d""" % (n_edges, n_classes, train_mask.numpy().sum(), val_mask.numpy().sum(), test_mask.numpy().sum())) # add self loop g = dgl.remove_self_loop(g) g = dgl.add_self_loop(g) n_edges = g.number_of_edges() # # normalization degs = tf.cast(tf.identity(g.in_degrees()), dtype=tf.float32) norm = tf.math.pow(degs, -0.5) norm = tf.where(tf.math.is_inf(norm), tf.zeros_like(norm), norm) g.ndata['norm'] = tf.expand_dims(norm, -1) # create GCN model model = GCN(g, in_feats, args.n_hidden, n_classes, args.n_layers, tf.nn.relu, args.dropout) optimizer = tf.keras.optimizers.Adam(learning_rate=args.lr) loss_fcn = tf.keras.losses.SparseCategoricalCrossentropy( from_logits=True) # initialize graph dur = [] for epoch in range(args.n_epochs): if epoch >= 3: t0 = time.time() # forward with tf.GradientTape() as tape: logits = model(features) loss_value = loss_fcn(labels[train_mask], logits[train_mask]) # Manually Weight Decay # We found Tensorflow has a different implementation on weight decay # of Adam(W) optimizer with PyTorch. And this results in worse results. # Manually adding weights to the loss to do weight decay solves this problem. for weight in model.trainable_weights: loss_value = loss_value + \ args.weight_decay*tf.nn.l2_loss(weight) grads = tape.gradient(loss_value, model.trainable_weights) optimizer.apply_gradients(zip(grads, model.trainable_weights)) if epoch >= 3: dur.append(time.time() - t0) acc = evaluate(model, features, labels, val_mask) print( "Epoch {:05d} | Time(s) {:.4f} | Loss {:.4f} | Accuracy {:.4f} | " "ETputs(KTEPS) {:.2f}".format(epoch, np.mean(dur), loss_value.numpy().item(), acc, n_edges / np.mean(dur) / 1000)) acc = evaluate(model, features, labels, test_mask) print("Test Accuracy {:.4f}".format(acc))
def main(args): # load and preprocess dataset if args.dataset == 'cora': data = CoraGraphDataset() elif args.dataset == 'citeseer': data = CiteseerGraphDataset() elif args.dataset == 'pubmed': data = PubmedGraphDataset() else: raise ValueError('Unknown dataset: {}'.format(args.dataset)) g = data[0] if args.gpu < 0: cuda = False else: cuda = True g = g.int().to(args.gpu) features = g.ndata['feat'] labels = g.ndata['label'] train_mask = g.ndata['train_mask'] val_mask = g.ndata['val_mask'] test_mask = g.ndata['test_mask'] in_feats = features.shape[1] n_classes = data.num_labels n_edges = data.graph.number_of_edges() print("""----Data statistics------' #Edges %d #Classes %d #Train samples %d #Val samples %d #Test samples %d""" % (n_edges, n_classes, train_mask.int().sum().item(), val_mask.int().sum().item(), test_mask.int().sum().item())) # add self loop if args.self_loop: g = dgl.remove_self_loop(g) g = dgl.add_self_loop(g) n_edges = g.number_of_edges() # normalization degs = g.in_degrees().float() norm = torch.pow(degs, -0.5) norm[torch.isinf(norm)] = 0 if cuda: norm = norm.cuda() g.ndata['norm'] = norm.unsqueeze(1) # create GCN model model = GCN(g, in_feats, args.n_hidden, n_classes, args.n_layers, F.relu, args.dropout) if cuda: model.cuda() loss_fcn = torch.nn.CrossEntropyLoss() # use optimizer optimizer = torch.optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.weight_decay) # initialize graph dur = [] for epoch in range(args.n_epochs): model.train() if epoch >= 3: t0 = time.time() # forward logits = model(features) loss = loss_fcn(logits[train_mask], labels[train_mask]) optimizer.zero_grad() loss.backward() optimizer.step() if epoch >= 3: dur.append(time.time() - t0) acc = evaluate(model, features, labels, val_mask) print( "Epoch {:05d} | Time(s) {:.4f} | Loss {:.4f} | Accuracy {:.4f} | " "ETputs(KTEPS) {:.2f}".format(epoch, np.mean(dur), loss.item(), acc, n_edges / np.mean(dur) / 1000)) print() acc = evaluate(model, features, labels, test_mask) print("Test accuracy {:.2%}".format(acc))
def main(args): # load and preprocess dataset if args.dataset == 'cora': data = CoraGraphDataset() elif args.dataset == 'citeseer': data = CiteseerGraphDataset() elif args.dataset == 'pubmed': data = PubmedGraphDataset() else: raise ValueError('Unknown dataset: {}'.format(args.dataset)) g = data[0] if args.gpu < 0: device = "/cpu:0" else: device = "/gpu:{}".format(args.gpu) g = g.to(device) with tf.device(device): features = g.ndata['feat'] labels = g.ndata['label'] train_mask = g.ndata['train_mask'] val_mask = g.ndata['val_mask'] test_mask = g.ndata['test_mask'] num_feats = features.shape[1] n_classes = data.num_labels n_edges = data.graph.number_of_edges() print("""----Data statistics------' #Edges %d #Classes %d #Train samples %d #Val samples %d #Test samples %d""" % (n_edges, n_classes, train_mask.numpy().sum(), val_mask.numpy().sum(), test_mask.numpy().sum())) g = dgl.remove_self_loop(g) g = dgl.add_self_loop(g) n_edges = g.number_of_edges() # create model heads = ([args.num_heads] * args.num_layers) + [args.num_out_heads] model = GAT(g, args.num_layers, num_feats, args.num_hidden, n_classes, heads, tf.nn.elu, args.in_drop, args.attn_drop, args.negative_slope, args.residual) print(model) if args.early_stop: stopper = EarlyStopping(patience=100) # loss_fcn = tf.keras.losses.SparseCategoricalCrossentropy( # from_logits=False) loss_fcn = tf.nn.sparse_softmax_cross_entropy_with_logits # use optimizer optimizer = tf.keras.optimizers.Adam(learning_rate=args.lr, epsilon=1e-8) # initialize graph dur = [] for epoch in range(args.epochs): if epoch >= 3: t0 = time.time() # forward with tf.GradientTape() as tape: tape.watch(model.trainable_weights) logits = model(features, training=True) loss_value = tf.reduce_mean( loss_fcn(labels=labels[train_mask], logits=logits[train_mask])) # Manually Weight Decay # We found Tensorflow has a different implementation on weight decay # of Adam(W) optimizer with PyTorch. And this results in worse results. # Manually adding weights to the loss to do weight decay solves this problem. for weight in model.trainable_weights: loss_value = loss_value + \ args.weight_decay*tf.nn.l2_loss(weight) grads = tape.gradient(loss_value, model.trainable_weights) optimizer.apply_gradients(zip(grads, model.trainable_weights)) if epoch >= 3: dur.append(time.time() - t0) train_acc = accuracy(logits[train_mask], labels[train_mask]) if args.fastmode: val_acc = accuracy(logits[val_mask], labels[val_mask]) else: val_acc = evaluate(model, features, labels, val_mask) if args.early_stop: if stopper.step(val_acc, model): break print( "Epoch {:05d} | Time(s) {:.4f} | Loss {:.4f} | TrainAcc {:.4f} |" " ValAcc {:.4f} | ETputs(KTEPS) {:.2f}".format( epoch, np.mean(dur), loss_value.numpy().item(), train_acc, val_acc, n_edges / np.mean(dur) / 1000)) print() if args.early_stop: model.load_weights('es_checkpoint.pb') acc = evaluate(model, features, labels, test_mask) print("Test Accuracy {:.4f}".format(acc))