class HIVDataset(Dataset): def __init__(self, name, pos_enc_dim=0, norm='none', verbose=True): start = time.time() if verbose: print("[I] Loading dataset %s..." % (name)) self.name = name self.dataset = DglGraphPropPredDataset(name='ogbg-molhiv') self.split_idx = self.dataset.get_idx_split() self.train = HIVDGL(self.dataset, self.split_idx['train'], norm=norm, pos_enc_dim=pos_enc_dim) self.val = HIVDGL(self.dataset, self.split_idx['valid'], norm=norm, pos_enc_dim=pos_enc_dim) self.test = HIVDGL(self.dataset, self.split_idx['test'], norm=norm, pos_enc_dim=pos_enc_dim) self.evaluator = Evaluator(name='ogbg-molhiv') if verbose: print('train, test, val sizes :', len(self.train), len(self.test), len(self.val)) print("[I] Finished loading.") print("[I] Data load time: {:.4f}s".format(time.time() - start)) # form a mini batch from a given list of samples = [(graph, label) pairs] def collate(self, samples): # The input samples is a list of pairs (graph, label). graphs, labels = map(list, zip(*samples)) labels = torch.cat(labels).long() tab_sizes_n = [graphs[i].number_of_nodes() for i in range(len(graphs))] tab_snorm_n = [ torch.FloatTensor(size, 1).fill_(1. / float(size)) for size in tab_sizes_n ] snorm_n = torch.cat(tab_snorm_n).sqrt() tab_sizes_e = [graphs[i].number_of_edges() for i in range(len(graphs))] tab_snorm_e = [ torch.FloatTensor(size, 1).fill_(1. / float(size)) for size in tab_sizes_e ] snorm_e = torch.cat(tab_snorm_e).sqrt() batched_graph = dgl.batch(graphs) return batched_graph, labels, snorm_n, snorm_e def _add_self_loops(self): # function for adding self loops # this function will be called only if self_loop flag is True self.train.graph_lists = [self_loop(g) for g in self.train.graph_lists] self.val.graph_lists = [self_loop(g) for g in self.val.graph_lists] self.test.graph_lists = [self_loop(g) for g in self.test.graph_lists]
def load_ogbg(name, device=th.device('cpu'), root='/home/eva_share_users/zhuyu'): from ogb.graphproppred import DglGraphPropPredDataset print('load', name) data = DglGraphPropPredDataset(name=name, root=root) #from IPython import embed; embed() from tqdm import tqdm out_channels = 0 for graph in tqdm(data): if name == 'ogbg-ppa': graph[0].ndata['feat'] = dgl.ops.copy_e_mean( graph[0], graph[0].edata['feat']) else: ef = graph[0].edata['feat'] edge = graph[0].edges()[1] H = th.zeros(graph[0].num_nodes(), 3) for i in range(graph[0].num_nodes()): mask = th.eq(edge, i) H[i, :] += th.matmul(mask.float(), ef.float()) H[i, :] /= graph[0].in_degrees(i) graph[0].ndata['feat'] = th.cat((graph[0].ndata['feat'], H), dim=1) #from IPython import embed; embed() in_channels = graph[0].ndata['feat'].shape[1] try: out_channels = max(out_channels, int(graph[1])) except: from IPython import embed embed() split_idx = data.get_idx_split() print('finish loading', name) from dgl.dataloading import GraphDataLoader train_loader = GraphDataLoader( data[split_idx['train']], batch_size=256, shuffle=True, ) valid_loader = GraphDataLoader( data[split_idx['valid']], batch_size=256, shuffle=True, ) test_loader = GraphDataLoader( data[split_idx['test']], batch_size=256, shuffle=True, ) #from IPython import embed; embed() return train_loader, valid_loader, test_loader, in_channels, out_channels + 1
class HIVDataset(Dataset): def __init__(self, name, verbose=True): start = time.time() if verbose: print("[I] Loading dataset %s..." % (name)) self.name = name self.dataset = DglGraphPropPredDataset(name = 'ogbg-molhiv') self.split_idx = self.dataset.get_idx_split() self.train = HIVDGL(self.dataset, self.split_idx['train']) self.val = HIVDGL(self.dataset, self.split_idx['valid']) self.test = HIVDGL(self.dataset, self.split_idx['test']) self.evaluator = Evaluator(name='ogbg-molhiv') if verbose: print('train, test, val sizes :', len(self.train), len(self.test), len(self.val)) print("[I] Finished loading.") print("[I] Data load time: {:.4f}s".format(time.time() - start)) # form a mini batch from a given list of samples = [(graph, label) pairs] def collate(self, samples): # The input samples is a list of pairs (graph, label). graphs, labels = map(list, zip(*samples)) labels = torch.cat(labels).long() batched_graph = dgl.batch(graphs) return batched_graph, labels def _add_self_loops(self): # function for adding self loops # this function will be called only if self_loop flag is True self.train.graph_lists = [self_loop(g) for g in self.train.graph_lists] self.val.graph_lists = [self_loop(g) for g in self.val.graph_lists] self.test.graph_lists = [self_loop(g) for g in self.test.graph_lists]
def run(args): from ogb.graphproppred import DglGraphPropPredDataset, Evaluator, collate_dgl from torch.utils.data import DataLoader dataset = DglGraphPropPredDataset(name="ogbg-molhiv") import os if not os.path.exists("heterographs.bin"): dataset.graphs = [hpno.heterograph(graph) for graph in dataset.graphs] from dgl.data.utils import save_graphs save_graphs("heterographs.bin", dataset.graphs) else: from dgl.data.utils import load_graphs dataset.graphs = load_graphs("heterographs.bin")[0] evaluator = Evaluator(name="ogbg-molhiv") in_features = 9 out_features = 1 split_idx = dataset.get_idx_split() train_loader = DataLoader(dataset[split_idx["train"]], batch_size=128, drop_last=True, shuffle=True, collate_fn=collate_dgl) valid_loader = DataLoader(dataset[split_idx["valid"]], batch_size=len(split_idx["valid"]), shuffle=False, collate_fn=collate_dgl) test_loader = DataLoader(dataset[split_idx["test"]], batch_size=len(split_idx["test"]), shuffle=False, collate_fn=collate_dgl) model = hpno.HierarchicalPathNetwork( in_features=in_features, out_features=args.hidden_features, hidden_features=args.hidden_features, depth=args.depth, readout=hpno.GraphReadout( in_features=args.hidden_features, out_features=out_features, hidden_features=args.hidden_features, ) ) if torch.cuda.is_available(): model = model.cuda() optimizer = torch.optim.Adam(model.parameters(), args.learning_rate, weight_decay=args.weight_decay) scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, "min", factor=0.5, patience=20) for idx_epoch in range(args.n_epochs): print(idx_epoch, flush=True) model.train() for g, y in train_loader: y = y.float() if torch.cuda.is_available(): g = g.to("cuda:0") y = y.cuda() optimizer.zero_grad() y_hat = model.forward(g, g.nodes['n1'].data["feat"].float()) loss = torch.nn.BCELoss()( input=y_hat.sigmoid(), target=y, ) loss.backward() optimizer.step() model.eval() with torch.no_grad(): g, y = next(iter(valid_loader)) y = y.float() if torch.cuda.is_available(): g = g.to("cuda:0") y = y.cuda() y_hat = model.forward(g, g.nodes['n1'].data["feat"].float()) loss = torch.nn.BCELoss()( input=y_hat.sigmoid(), target=y, ) scheduler.step(loss) if optimizer.param_groups[0]["lr"] <= 0.01 * args.learning_rate: break model = model.cpu() g, y = next(iter(valid_loader)) rocauc_vl = evaluator.eval( { "y_true": y.float(), "y_pred": model.forward(g, g.nodes['n1'].data["feat"].float()).sigmoid() } )["rocauc"] g, y = next(iter(test_loader)) rocauc_te = evaluator.eval( { "y_true": y.float(), "y_pred": model.forward(g, g.nodes['n1'].data["feat"].float()).sigmoid() } )["rocauc"] import pandas as pd df = pd.DataFrame( { args.data: { "rocauc_te": rocauc_te, "rocauc_vl": rocauc_vl, } } ) df.to_csv("%s.csv" % args.out)
def main(): parser = argparse.ArgumentParser(description='OGBN-MolHiv') parser.add_argument('--device', type=int, default=0) parser.add_argument('--num_workers', type=int, default=4) parser.add_argument('--log_steps', type=int, default=1) parser.add_argument('--batch_size', type=int, default=64) parser.add_argument('--num_layers', type=int, default=5) parser.add_argument('--emb_dim', type=int, default=256) parser.add_argument('--dropout', type=float, default=0.5) parser.add_argument('--lr', type=float, default=0.001) parser.add_argument('--epochs', type=int, default=50) parser.add_argument('--runs', type=int, default=10) parser.add_argument('--eval', action='store_true', help='If not set, we will only do the training part.') parser.add_argument('--eval_batch_size', type=int, default=2048) args = parser.parse_args() print(args) device = f'cuda:{args.device}' if torch.cuda.is_available() else 'cpu' device = torch.device(device) dataset = DglGraphPropPredDataset(name='ogbg-molhiv') split_idx = dataset.get_idx_split() evaluator = Evaluator(name='ogbg-molhiv') train_loader = GraphDataLoader(dataset[split_idx["train"]], batch_size=args.batch_size, shuffle=True, num_workers=args.num_workers) val_loader = GraphDataLoader(dataset[split_idx["valid"]], batch_size=args.eval_batch_size, shuffle=True, num_workers=0) test_loader = GraphDataLoader(dataset[split_idx["test"]], batch_size=args.eval_batch_size, shuffle=True, num_workers=0) model = GCN(args.emb_dim, num_classes=dataset.num_tasks, num_layers=args.num_layers, dropout=args.dropout).to(device) logger = Logger(args.runs, args) dur = [] for run in range(args.runs): model.reset_parameters() optimizer = torch.optim.Adam(model.parameters(), lr=args.lr) for epoch in range(1, args.epochs + 1): t0 = time.time() loss = train(model, device, train_loader, optimizer) if epoch >= 3: dur.append(time.time() - t0) print('Training time/epoch {}'.format(np.mean(dur))) if not args.eval: continue val_rocauc = test(model, device, val_loader, evaluator)[dataset.eval_metric] test_rocauc = test(model, device, test_loader, evaluator)[dataset.eval_metric] logger.add_result(run, (0.0, val_rocauc, test_rocauc)) if epoch % args.log_steps == 0: print(f'Run: {run + 1:02d}, ' f'Epoch: {epoch:02d}, ' f'Loss: {loss:.4f}, ' f'Valid: {val_rocauc:.4f} ' f'Test: {test_rocauc:.4f}') if args.eval: logger.print_statistics(run) if args.eval: logger.print_statistics()
def train_molhiv(args, device, metrics_dict): dataset = DglGraphPropPredDataset(name='ogbg-molhiv') split_idx = dataset.get_idx_split() train_loader = DataLoader(dataset[split_idx["train"]], batch_size=32, shuffle=True, collate_fn=collate_dgl) val_loader = DataLoader(dataset[split_idx["valid"]], batch_size=32, shuffle=False, collate_fn=collate_dgl) test_loader = DataLoader(dataset[split_idx["test"]], batch_size=32, shuffle=False, collate_fn=collate_dgl) model = globals()[args.model_type]( node_dim=dataset[0][0].ndata['feat'].shape[1], edge_dim=dataset[0][0].edata['feat'].shape[1] if args.use_e_features else 0, **args.model_parameters) print('model trainable params: ', sum(p.numel() for p in model.parameters() if p.requires_grad)) collate_function = globals()[ args.collate_function] if args.collate_params == {} else globals()[ args.collate_function](**args.collate_params) metrics = {metric: metrics_dict[metric] for metric in args.metrics} tensorboard_functions = { function: TENSORBOARD_FUNCTIONS[function] for function in args.tensorboard_functions } # Needs "from torch.optim import *" and "from models import *" to work transferred_params = [ v for k, v in model.named_parameters() if any(transfer_name in k for transfer_name in args.transfer_layers) ] new_params = [ v for k, v in model.named_parameters() if all( transfer_name not in k for transfer_name in args.transfer_layers) ] transfer_lr = args.optimizer_params[ 'lr'] if args.transferred_lr == None else args.transferred_lr optim = globals()[args.optimizer]([{ 'params': new_params }, { 'params': transferred_params, 'lr': transfer_lr }], **args.optimizer_params) trainer = Trainer(model=model, args=args, metrics=metrics, main_metric=args.main_metric, main_metric_goal=args.main_metric_goal, optim=optim, loss_func=globals()[args.loss_func](**args.loss_params), device=device, tensorboard_functions=tensorboard_functions, scheduler_step_per_batch=args.scheduler_step_per_batch) trainer.train(train_loader, val_loader) if args.eval_on_test: trainer.evaluation(test_loader, data_split='test')
def main(): # check cuda device = f'cuda:{args.gpu}' if args.gpu >= 0 and torch.cuda.is_available( ) else 'cpu' # load ogb dataset & evaluator dataset = DglGraphPropPredDataset(name=args.dataset) evaluator = Evaluator(name=args.dataset) g, _ = dataset[0] edge_feat_dim = g.edata['feat'].size()[-1] n_classes = int(dataset.num_classes) split_idx = dataset.get_idx_split() train_loader = DataLoader(dataset[split_idx["train"]], batch_size=args.batch_size, shuffle=True, collate_fn=collate_dgl) valid_loader = DataLoader(dataset[split_idx["valid"]], batch_size=args.batch_size, shuffle=False, collate_fn=collate_dgl) test_loader = DataLoader(dataset[split_idx["test"]], batch_size=args.batch_size, shuffle=False, collate_fn=collate_dgl) # load model model = DeeperGCN(dataset=args.dataset, node_feat_dim=edge_feat_dim, edge_feat_dim=edge_feat_dim, hid_dim=args.hid_dim, out_dim=n_classes, num_layers=args.num_layers, dropout=args.dropout, norm=args.norm, beta=args.beta, mlp_layers=args.mlp_layers).to(device) print(model) opt = optim.Adam(model.parameters(), lr=args.lr) # training & validation & testing best_acc = 0 best_model = copy.deepcopy(model) print('---------- Training ----------') for i in range(args.epochs): train_loss = train(model, device, train_loader, opt) if i % args.eval_steps == 0: train_acc = test(model, device, train_loader, evaluator) valid_acc = test(model, device, valid_loader, evaluator) print( f'Epoch {i} | Train Loss: {train_loss:.4f} | Train Acc: {train_acc:.4f} | Valid Acc: {valid_acc:.4f}' ) if valid_acc > best_acc: best_acc = valid_acc best_model = copy.deepcopy(model) else: print(f'Epoch {i} | Train Loss: {train_loss:.4f}') print('---------- Testing ----------') test_acc = test(best_model, device, test_loader, evaluator) print(f'Test Acc: {test_acc}')
def main(args): torch.manual_seed(args.seed) np.random.seed(args.seed) device = torch.device( "cuda:" + str(args.device)) if torch.cuda.is_available() else torch.device("cpu") # Load dataset and evaluator dataset = DglGraphPropPredDataset(name=args.dataset) split_idx = dataset.get_idx_split() evaluator = Evaluator(args.dataset) if args.pos_enc_dim > 0: # Add graph positional encodings print("Adding PEs...") dataset.graphs = [ add_positional_encoding(g, args.pos_enc_dim) for g in tqdm(dataset.graphs) ] # Basic pre-processing if args.dataset == 'ogbg-molpcba': print("Removing training graphs with 0 edges...") train_split = [] for idx, g in enumerate(tqdm(dataset.graphs)): if idx in split_idx["train"] and g.number_of_edges() != 0: train_split.append(idx) split_idx["train"] = torch.LongTensor(train_split) # Prepare dataloaders train_loader = DataLoader(dataset[split_idx["train"]], batch_size=args.batch_size, shuffle=True, num_workers=args.num_workers, collate_fn=collate_dgl) valid_loader = DataLoader(dataset[split_idx["valid"]], batch_size=args.batch_size, shuffle=False, num_workers=args.num_workers, collate_fn=collate_dgl) test_loader = DataLoader(dataset[split_idx["test"]], batch_size=args.batch_size, shuffle=False, num_workers=args.num_workers, collate_fn=collate_dgl) # Initialize model, optimizer and scheduler if args.gnn in ['gated-gcn', 'gcn', 'mlp']: model = GNN_mol(gnn_type=args.gnn, num_tasks=dataset.num_tasks, num_layer=args.num_layer, emb_dim=args.emb_dim, dropout=args.dropout, batch_norm=True, residual=True, pos_enc_dim=args.pos_enc_dim, graph_pooling=args.pooling, virtualnode=args.virtualnode) model.to(device) print(model) total_param = 0 for param in model.parameters(): total_param += np.prod(list(param.data.size())) print(f'Total parameters: {total_param}') optimizer = optim.Adam(model.parameters(), lr=args.lr) scheduler = optim.lr_scheduler.ReduceLROnPlateau( optimizer, mode='min', factor=args.lr_reduce_factor, patience=args.lr_scheduler_patience, verbose=True) else: raise ValueError('Invalid GNN type') # Define loss function cls_criterion = torch.nn.BCEWithLogitsLoss() # Create Tensorboard logger start_time_str = time.strftime("%Y%m%dT%H%M%S") log_dir = os.path.join( "logs", args.dataset, f"{args.expt_name}-{args.gnn}-L{args.num_layer}-h{args.emb_dim}-d{args.dropout}-LR{args.lr}", f"{start_time_str}-GPU{args.device}") tb_logger = SummaryWriter(log_dir) # Training loop train_curve = [] valid_curve = [] test_curve = [] for epoch in range(1, args.epochs + 1): print("=====Epoch {}".format(epoch)) tb_logger.add_scalar('lr', optimizer.param_groups[0]['lr'], epoch) print('Training...') train(model, device, train_loader, optimizer, cls_criterion) print('Evaluating...') train_loss, train_perf = eval(model, device, train_loader, evaluator, cls_criterion) valid_loss, valid_perf = eval(model, device, valid_loader, evaluator, cls_criterion) _, test_perf = eval(model, device, test_loader, evaluator, cls_criterion) # Log statistics to Tensorboard, etc. print({ 'Train': train_perf, 'Validation': valid_perf, 'Test': test_perf }) tb_logger.add_scalar('loss/train', train_loss, epoch) tb_logger.add_scalar(f'{dataset.eval_metric}/train', train_perf[dataset.eval_metric], epoch) tb_logger.add_scalar('loss/valid', valid_loss, epoch) tb_logger.add_scalar(f'{dataset.eval_metric}/valid', valid_perf[dataset.eval_metric], epoch) tb_logger.add_scalar(f'{dataset.eval_metric}/test', test_perf[dataset.eval_metric], epoch) train_curve.append(train_perf[dataset.eval_metric]) valid_curve.append(valid_perf[dataset.eval_metric]) test_curve.append(test_perf[dataset.eval_metric]) if args.lr_scheduler_patience > 0: # Reduce LR using scheduler scheduler.step(valid_loss) if 'classification' in dataset.task_type: best_val_epoch = np.argmax(np.array(valid_curve)) best_train = max(train_curve) else: best_val_epoch = np.argmin(np.array(valid_curve)) best_train = min(train_curve) print('Finished training!') print('Best validation score: {}'.format(valid_curve[best_val_epoch])) print('Test score: {}'.format(test_curve[best_val_epoch])) torch.save( { 'args': args, 'model': model.__repr__, 'total_param': total_param, 'BestEpoch': best_val_epoch, 'Validation': valid_curve[best_val_epoch], 'Test': test_curve[best_val_epoch], 'Train': train_curve[best_val_epoch], 'BestTrain': best_train, }, os.path.join(log_dir, "results.pt"))
def main(): # Training settings parser = argparse.ArgumentParser( description='GNN baselines on ogbgmol* data with Pytorch Geometrics') parser.add_argument('--device', type=int, default=0, help='which gpu to use if any (default: 0)') parser.add_argument( '--gnn', type=str, default='gin-virtual', help= 'GNN gin, gin-virtual, or gcn, or gcn-virtual (default: gin-virtual)') parser.add_argument('--drop_ratio', type=float, default=0.5, help='dropout ratio (default: 0.5)') parser.add_argument( '--num_layer', type=int, default=5, help='number of GNN message passing layers (default: 5)') parser.add_argument( '--emb_dim', type=int, default=512, help='dimensionality of hidden units in GNNs (default: 300)') parser.add_argument('--batch_size', type=int, default=32, help='input batch size for training (default: 32)') parser.add_argument('--epochs', type=int, default=100, help='number of epochs to train (default: 100)') parser.add_argument('--num_workers', type=int, default=0, help='number of workers (default: 0)') parser.add_argument('--dataset', type=str, default="molhiv", help='dataset name (default: ogbg-molhiv)') parser.add_argument( '--rank', type=int, default=512, help='dimensionality of rank units in GNNs (default: 300)') parser.add_argument('--filename', type=str, default="", help='filename to output result (default: )') parser.add_argument('--lr', type=float, default=0.003) parser.add_argument('--wd', type=float, default=5e-5, help='Weight decay (L2 loss on parameters).') args = parser.parse_args() device = torch.device( "cuda:" + str(args.device)) if torch.cuda.is_available() else torch.device("cpu") ### automatic dataloading and splitting dataset = DglGraphPropPredDataset(name="ogbg-" + args.dataset, root='torch_geometric_data/') split_idx = dataset.get_idx_split() ### automatic evaluator. takes dataset name as input evaluator = Evaluator(name="ogbg-" + args.dataset) train_loader = DataLoader(dataset[split_idx["train"]], batch_size=args.batch_size, shuffle=True, num_workers=args.num_workers, collate_fn=collate_dgl) valid_loader = DataLoader(dataset[split_idx["valid"]], batch_size=args.batch_size, shuffle=False, num_workers=args.num_workers, collate_fn=collate_dgl) test_loader = DataLoader(dataset[split_idx["test"]], batch_size=args.batch_size, shuffle=False, num_workers=args.num_workers, collate_fn=collate_dgl) model = GNN(num_tasks=dataset.num_tasks, num_layer=args.num_layer, emb_dim=args.emb_dim, rank=args.rank, drop_ratio=args.drop_ratio).to(device) optimizer = optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.wd) valid_curve = [] test_curve = [] train_curve = [] for epoch in range(1, args.epochs + 1): print("=====Epoch {}".format(epoch)) print('Training...') train(model, device, train_loader, optimizer, dataset.task_type) print('Evaluating...') train_perf = eval(model, device, train_loader, evaluator) valid_perf = eval(model, device, valid_loader, evaluator) test_perf = eval(model, device, test_loader, evaluator) print({ 'Train': train_perf, 'Validation': valid_perf, 'Test': test_perf }) train_curve.append(train_perf[dataset.eval_metric]) valid_curve.append(valid_perf[dataset.eval_metric]) test_curve.append(test_perf[dataset.eval_metric]) if 'classification' in dataset.task_type: best_val_epoch = np.argmax(np.array(valid_curve)) best_train = max(train_curve) else: best_val_epoch = np.argmin(np.array(valid_curve)) best_train = min(train_curve) print('Finished training!') print('Best validation score: {}'.format(valid_curve[best_val_epoch])) print('Test score: {}'.format(test_curve[best_val_epoch])) if not args.filename == '': torch.save( { 'Val': valid_curve[best_val_epoch], 'Test': test_curve[best_val_epoch], 'Train': train_curve[best_val_epoch], 'BestTrain': best_train }, args.filename)
def main(): # Training settings parser = argparse.ArgumentParser( description='GNN baselines on ogbg-ppa with DGL') parser.add_argument( '--gnn', type=str, default='gin-virtual', help='GNN gin, gcn, gin-virtual, gcn-virtual (default: gin-virtual)') parser.add_argument('--dropout', type=float, default=0.5, help='dropout ratio (default: 0.5)') parser.add_argument( '--n_layers', type=int, default=5, help='number of GNN message passing layers (default: 5)') parser.add_argument('--hidden_feats', type=int, default=300, help='number of hidden units in GNNs (default: 300)') parser.add_argument('--batch_size', type=int, default=32, help='input batch size for training (default: 32)') parser.add_argument('--epochs', type=int, default=100, help='number of epochs for training (default: 100)') parser.add_argument('--num_workers', type=int, default=0, help='number of workers (default: 0)') parser.add_argument('--dataset', type=str, default="ogbg-ppa", help='dataset name (default: ogbg-ppa)') parser.add_argument('--filename', type=str, help='filename to output result') args = parser.parse_args() if args.filename is None: args.filename = args.gnn if torch.cuda.is_available(): device = torch.device('cuda:0') else: device = torch.device('cpu') # data loading and splitting dataset = DglGraphPropPredDataset(name=args.dataset) # initialize node features for i in range(len(dataset)): dataset[i][0].ndata['h'] = torch.zeros( dataset[i][0].number_of_nodes()).long() splitted_idx = dataset.get_idx_split() # automatic evaluator taking dataset name as input evaluator = Evaluator(args.dataset) # using collate_dgl train_loader = DataLoader(dataset[splitted_idx["train"]], batch_size=args.batch_size, shuffle=True, collate_fn=collate_dgl, num_workers=args.num_workers) valid_loader = DataLoader(dataset[splitted_idx["valid"]], batch_size=args.batch_size, shuffle=False, collate_fn=collate_dgl, num_workers=args.num_workers) test_loader = DataLoader(dataset[splitted_idx["test"]], batch_size=args.batch_size, shuffle=False, collate_fn=collate_dgl, num_workers=args.num_workers) if args.gnn == 'gin': gnn_type = 'gin' virtual_node = False if args.gnn == 'gcn': gnn_type = 'gcn' virtual_node = False if args.gnn == 'gin-virtual': gnn_type = 'gin' virtual_node = True if args.gnn == 'gcn-virtual': gnn_type = 'gcn' virtual_node = True model = GNNOGBPredictor( in_edge_feats=dataset[0][0].edata['feat'].shape[-1], hidden_feats=args.hidden_feats, n_layers=args.n_layers, n_tasks=int(dataset.num_classes), dropout=args.dropout, gnn_type=gnn_type, virtual_node=virtual_node).to(device) criterion = torch.nn.CrossEntropyLoss() optimizer = optim.Adam(model.parameters(), lr=0.001) valid_curve = [] test_curve = [] train_curve = [] time_curve = [] for epoch in range(1, args.epochs + 1): print("=====Epoch {}".format(epoch)) print('Training...') t0 = time.time() train(model, device, train_loader, criterion, optimizer) t1 = time.time() if epoch >= 3: time_curve.append(t1 - t0) print('Evaluating...') train_perf = eval(model, device, train_loader, evaluator) valid_perf = eval(model, device, valid_loader, evaluator) test_perf = eval(model, device, test_loader, evaluator) print({ 'Train': train_perf, 'Validation': valid_perf, 'Test': test_perf }) if epoch >= 3: print('Training Time: ', time_curve[-1]) train_curve.append(train_perf['acc']) valid_curve.append(valid_perf['acc']) test_curve.append(test_perf['acc']) best_val_epoch = np.argmax(np.array(valid_curve)) best_train = max(train_curve) print('Finished training!') print('Best validation score: {}'.format(valid_curve[best_val_epoch])) print('Test score: {}'.format(test_curve[best_val_epoch])) print('Avg Training Time: ', np.mean(time_curve)) if not args.filename == '': torch.save( { 'Val': valid_curve[best_val_epoch], 'Test': test_curve[best_val_epoch], 'Train': train_curve[best_val_epoch], 'BestTrain': best_train }, args.filename)
def main(): # Training settings parser = argparse.ArgumentParser( description='GNN baselines on ogbg-mol* data with DGL') parser.add_argument('--device', type=int, default=0, help='which gpu to use if any (default: 0)') parser.add_argument( '--gnn', type=str, default='gcn', help= 'gin, or gin-virtual, or gcn, or gcn-virtual (default: gin-virtual)') parser.add_argument('--drop_ratio', type=float, default=0.5, help='dropout ratio (default: 0.5)') parser.add_argument( '--n_layers', type=int, default=5, help='number of GNN message passing layers (default: 5)') parser.add_argument( '--n_hidden', type=int, default=300, help='dimensionality of hidden units in GNNs (default: 300)') parser.add_argument('--batch_size', type=int, default=32, help='input batch size for training (default: 32)') parser.add_argument('--epochs', type=int, default=100, help='number of epochs to train (default: 100)') parser.add_argument('--num_workers', type=int, default=0, help='number of workers (default: 0)') parser.add_argument('--dataset', type=str, default="ogbg-molhiv", help='dataset name (default: ogbg-molhiv)') parser.add_argument('--graph_pooling_type', type=str, default="mean", choices=["sum", "mean", "max"], help='type of graph pooling: sum, mean or max') parser.add_argument('--feature', type=str, default="full", choices=["full", "simple"], help='full feature or simple feature') parser.add_argument( '--filename', type=str, default="gin-virtual.pth", help='filename to output result (default: gin-virtual.pth)') args = parser.parse_args() if torch.cuda.is_available(): device = torch.device("cuda:" + str(args.device)) torch.cuda.set_device(args.device) else: device = torch.device("cpu") ### dataloading and splitting dataset = DglGraphPropPredDataset(name=args.dataset) print('Metric', dataset.eval_metric) if args.feature == 'simple': print('using simple feature') else: print('using full feature') splitted_idx = dataset.get_idx_split() ### automatic evaluator. takes dataset name as input evaluator = Evaluator(args.dataset) # using collate_dgl train_loader = DataLoader(dataset[splitted_idx["train"]], batch_size=args.batch_size, shuffle=True, collate_fn=collate_dgl, num_workers=args.num_workers) valid_loader = DataLoader(dataset[splitted_idx["valid"]], batch_size=args.batch_size, shuffle=False, collate_fn=collate_dgl, num_workers=args.num_workers) test_loader = DataLoader(dataset[splitted_idx["test"]], batch_size=args.batch_size, shuffle=False, collate_fn=collate_dgl, num_workers=args.num_workers) if args.gnn == 'gin': gnn_type = 'gin' virtual_node = False if args.gnn == 'gcn': gnn_type = 'gcn' virtual_node = False if args.gnn == 'gin-virtual': gnn_type = 'gin' virtual_node = True if args.gnn == 'gcn-virtual': gnn_type = 'gcn' virtual_node = True model = GNNOGBPredictor( in_edge_feats=dataset[0][0].edata['feat'].shape[-1], hidden_feats=args.hidden_feats, n_layers=args.n_layers, n_tasks=int(dataset.num_classes), dropout=args.dropout, gnn_type=gnn_type, virtual_node=virtual_node).to(device) optimizer = optim.Adam(model.parameters(), lr=0.001) model = model.to(device) valid_curve = [] test_curve = [] train_curve = [] time_curve = [] for epoch in range(1, args.epochs + 1): print("=====Epoch {}".format(epoch)) print('Training...') t0 = time.time() train(args.gnn, model, args.feature, device, train_loader, optimizer, dataset.task_type) if epoch >= 3: t1 = time.time() time_curve.append(t1 - t0) print('Training Time: ', time_curve[-1]) print('Evaluating...') train_perf = eval(args.gnn, model, args.feature, device, train_loader, evaluator, dataset.task_type) valid_perf = eval(args.gnn, model, args.feature, device, valid_loader, evaluator, dataset.task_type) test_perf = eval(args.gnn, model, args.feature, device, test_loader, evaluator, dataset.task_type) print({ 'Train': train_perf, 'Validation': valid_perf, 'Test': test_perf }) # dataset.eval_metric = 'rocauc' train_curve.append(train_perf[dataset.eval_metric]) valid_curve.append(valid_perf[dataset.eval_metric]) test_curve.append(test_perf[dataset.eval_metric]) if 'classification' in dataset.task_type: best_val_epoch = np.argmax(np.array(valid_curve)) best_train = max(train_curve) else: best_val_epoch = np.argmin(np.array(valid_curve)) best_train = min(train_curve) print('Finished training!') print('Best validation score: {}'.format(valid_curve[best_val_epoch])) print('Test score: {}'.format(test_curve[best_val_epoch])) print('Avg Training Time: ', np.mean(time_curve)) if not args.filename == '': torch.save( { 'Val': valid_curve[best_val_epoch], 'Test': test_curve[best_val_epoch], 'Train': train_curve[best_val_epoch], 'BestTrain': best_train }, args.filename)
def main(): # Training settings parser = argparse.ArgumentParser( description='GNN baselines on ogbgmol* data with DGL') parser.add_argument('--device', type=int, default=0, help='which gpu to use if any (default: 0)') parser.add_argument('--gnn', type=str, default='Cheb_net', help='GNN (default: Cheb_net)') parser.add_argument('--dropout', type=float, default=0.5, help='dropout ratio (default: 0.5)') parser.add_argument( '--num_layer', type=int, default=5, help='number of GNN message passing layers (default: 5)') parser.add_argument( '--emb_dim', type=int, default=300, help='dimensionality of hidden units in GNNs (default: 300)') parser.add_argument('--batch_size', type=int, default=32, help='input batch size for training (default: 32)') parser.add_argument('--epochs', type=int, default=100, help='number of epochs to train (default: 100)') parser.add_argument('--lr', type=float, default=1e-3, help='learning rate (default: 1e-3)') parser.add_argument('--num_workers', type=int, default=0, help='number of workers (default: 0)') parser.add_argument('--dataset', type=str, default="ogbg-molhiv", help='dataset name (default: ogbg-molhiv)') parser.add_argument('--filename', type=str, default="", help='filename to output result (default: )') args = parser.parse_args() device = torch.device( "cuda:" + str(args.device)) if torch.cuda.is_available() else torch.device("cpu") ### automatic dataloading and splitting dataset = DglGraphPropPredDataset(name=args.dataset) if not os.path.exists('results'): os.makedirs('results') writer = SummaryWriter(log_dir='results/' + args.filename + 'logs/' + args.dataset + '/' + args.gnn) split_idx = dataset.get_idx_split() ### automatic evaluator. takes dataset name as input evaluator = Evaluator(args.dataset) train_loader = DataLoader(dataset[split_idx["train"]], batch_size=args.batch_size, shuffle=True, num_workers=args.num_workers, collate_fn=collate_dgl, pin_memory=True) valid_loader = DataLoader(dataset[split_idx["valid"]], batch_size=args.batch_size, shuffle=False, num_workers=args.num_workers, collate_fn=collate_dgl, pin_memory=True) test_loader = DataLoader(dataset[split_idx["test"]], batch_size=args.batch_size, shuffle=False, num_workers=args.num_workers, collate_fn=collate_dgl, pin_memory=True) if args.gnn in ['gated-gcn', 'mlp', 'Cheb_net']: model = GNN(gnn_type=args.gnn, num_tasks=dataset.num_tasks, num_layer=args.num_layer, emb_dim=args.emb_dim, dropout=args.dropout, batch_norm=True, residual=True, graph_pooling="mean") model.to(device) else: raise ValueError('Invalid GNN type') print(model) total_param = 0 for param in model.parameters(): total_param += np.prod(list(param.data.size())) print(f'Total parameters: {total_param}') optimizer = optim.Adam(model.parameters(), lr=args.lr) valid_curve = [] test_curve = [] train_curve = [] for epoch in range(1, args.epochs + 1): print("=====Epoch {}".format(epoch)) print('Training...') train(model, device, train_loader, optimizer, dataset.task_type) print('Evaluating...') train_perf = eval(model, device, train_loader, evaluator) valid_perf = eval(model, device, valid_loader, evaluator) test_perf = eval(model, device, test_loader, evaluator) print({ 'Train': train_perf, 'Validation': valid_perf, 'Test': test_perf }) train_curve.append(train_perf[dataset.eval_metric]) valid_curve.append(valid_perf[dataset.eval_metric]) test_curve.append(test_perf[dataset.eval_metric]) writer.add_scalar('Val', valid_perf[dataset.eval_metric], epoch) writer.add_scalar('Test', test_perf[dataset.eval_metric], epoch) writer.add_scalar('Train', train_perf[dataset.eval_metric], epoch) if 'classification' in dataset.task_type: best_val_epoch = np.argmax(np.array(valid_curve)) best_train = max(train_curve) else: best_val_epoch = np.argmin(np.array(valid_curve)) best_train = min(train_curve) print('Finished training!') print('Best validation score: {}'.format(valid_curve[best_val_epoch])) print('Test score: {}'.format(test_curve[best_val_epoch])) if not args.filename == '': torch.save( { 'Val': valid_curve[best_val_epoch], 'Test': test_curve[best_val_epoch], 'Train': train_curve[best_val_epoch], 'BestTrain': best_train }, args.filename) writer.add_scalar('Best Val', valid_curve[best_val_epoch], best_val_epoch) writer.add_scalar('Best Test', test_curve[best_val_epoch], best_val_epoch) writer.add_scalar('Best Train', train_curve[best_val_epoch], best_val_epoch) writer.add_scalar('BestTrain', best_train) writer.close()
def main(): parser = argparse.ArgumentParser( description= 'PyTorch graph convolutional neural net for whole-graph classification' ) parser.add_argument('--dataset', type=str, default="MUTAG", help='name of dataset (default: MUTAG)') parser.add_argument('--device', type=int, default=0, help='which gpu to use if any (default: 0)') parser.add_argument('--batch_size', type=int, default=32, help='input batch size for training (default: 32)') parser.add_argument( '--iters_per_epoch', type=int, default=50, help='number of iterations per each epoch (default: 50)') parser.add_argument('--epochs', type=int, default=350, help='number of epochs to train (default: 350)') parser.add_argument('--lr', type=float, default=0.01, help='learning rate (default: 0.01)') parser.add_argument( '--seed', type=int, default=0, help='random seed for splitting the dataset into 10 (default: 0)') parser.add_argument('--hidden_dim', type=int, default=3, help='number of hidden units (default: 64)') parser.add_argument('--rank_dim', type=int, default=10, help='number of hidden units (default: 64)') parser.add_argument('--final_dropout', type=float, default=0.5, help='final layer dropout (default: 0.5)') parser.add_argument( '--graph_pooling_type', type=str, default="sum", choices=["sum", "average"], help='Pooling for over nodes in a graph: sum or average') parser.add_argument( '--neighbor_pooling_type', type=str, default="sum", choices=["sum", "average", "max"], help='Pooling for over neighboring nodes: sum, average or max') parser.add_argument( '--learn_eps', action="store_true", help= 'Whether to learn the epsilon weighting for the center nodes. Does not affect training accuracy though.' ) parser.add_argument( '--degree_as_tag', action="store_true", help= 'let the input node features be the degree of nodes (heuristics for unlabeled graph)' ) parser.add_argument('--filename', type=str, default="", help='output file') args = parser.parse_args() #set up seeds and gpu device torch.manual_seed(0) np.random.seed(0) device = torch.device( "cuda:" + str(args.device)) if torch.cuda.is_available() else torch.device("cpu") if torch.cuda.is_available(): torch.cuda.manual_seed_all(0) #graphs, num_classes = load_data(args.dataset, args.degree_as_tag) dataset = DglGraphPropPredDataset(name="ogbg-" + args.dataset, root='torch_geometric_data/') split_idx = dataset.get_idx_split() train_graphs = dataset[split_idx["train"]] valid_graphs = dataset[split_idx["valid"]] test_graphs = dataset[split_idx["test"]] num_classes = (torch.max( torch.LongTensor([dataset[idx][1] for idx in range(len(dataset))])) + 1).numpy() ##10-fold cross validation. Conduct an experiment on the fold specified by args.fold_idx. #train_graphs, test_graphs = separate_data(graphs, args.seed, args.fold_idx) model = GraphCPPooling(train_graphs[0][0].ndata['feat'].shape[1], args.hidden_dim, args.rank_dim, num_classes, args.final_dropout, device).to(device) optimizer = optim.Adam(model.parameters(), lr=args.lr) scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=50, gamma=0.5) evaluator = Evaluator(name="ogbg-" + args.dataset) patience = 50 vacc_mx = 0.0 curr_step = 0 for epoch in range(1, args.epochs + 1): scheduler.step() avg_loss = train(args, model, device, train_graphs, optimizer, epoch) train_acc, val_acc = validation(args, model, evaluator, device, train_graphs, valid_graphs, epoch, args.dataset) if val_acc >= vacc_mx: #or loss_accum <= vlss_mn: #if val_acc >= vacc_mx and loss_accum <= vlss_mn: curr_step = 0 best_test = test(args, model, evaluator, device, valid_graphs, epoch, args.dataset) vacc_mx = val_acc #vlss_mn = np.min((loss_accum, vlss_mn)) print("Best val: %.4f, best test: %.4f" % (vacc_mx, best_test)) else: curr_step += 1 if curr_step >= patience: break
class HIVDataset(Dataset): def __init__(self, name, pos_enc_dim=0, norm='none', path='dataset/ogbg-molhiv', directions=['subgraphs'], verbose=True, **subgraph_params): start = time.time() if verbose: print("[I] Loading dataset %s..." % (name)) self.name = name ##### MODIFIED CODE HERE if 'subgraphs' in directions: self.dataset, self.split_idx = prepare_dataset( path, name, **subgraph_params) print("One hot encoding substructure counts... ", end='') self.dataset, self.d_id = encode(self.dataset, subgraph_params['id_encoding']) else: self.dataset = DglGraphPropPredDataset(name=name, root=path) self.split_idx = self.dataset.get_idx_split() self.d_id = None self.train = HIVDGL(self.dataset, self.split_idx['train'], norm=norm, pos_enc_dim=pos_enc_dim, directions=directions, **subgraph_params) self.val = HIVDGL(self.dataset, self.split_idx['valid'], norm=norm, pos_enc_dim=pos_enc_dim, directions=directions, **subgraph_params) self.test = HIVDGL(self.dataset, self.split_idx['test'], norm=norm, pos_enc_dim=pos_enc_dim, directions=directions, **subgraph_params) ##### MODIFIED CODE HERE #import pdb;pdb.set_trace() self.evaluator = Evaluator(name='ogbg-molhiv') if verbose: print('train, test, val sizes :', len(self.train), len(self.test), len(self.val)) print("[I] Finished loading.") print("[I] Data load time: {:.4f}s".format(time.time() - start)) # form a mini batch from a given list of samples = [(graph, label) pairs] def collate(self, samples): # The input samples is a list of pairs (graph, label). graphs, labels = map(list, zip(*samples)) labels = torch.cat(labels).long() tab_sizes_n = [graphs[i].number_of_nodes() for i in range(len(graphs))] tab_snorm_n = [ torch.FloatTensor(size, 1).fill_(1. / float(size)) for size in tab_sizes_n ] snorm_n = torch.cat(tab_snorm_n).sqrt() tab_sizes_e = [graphs[i].number_of_edges() for i in range(len(graphs))] tab_snorm_e = [ torch.FloatTensor(size, 1).fill_(1. / float(size)) for size in tab_sizes_e ] snorm_e = torch.cat(tab_snorm_e).sqrt() batched_graph = dgl.batch(graphs) return batched_graph, labels, snorm_n, snorm_e def _add_self_loops(self): # function for adding self loops # this function will be called only if self_loop flag is True self.train.graph_lists = [self_loop(g) for g in self.train.graph_lists] self.val.graph_lists = [self_loop(g) for g in self.val.graph_lists] self.test.graph_lists = [self_loop(g) for g in self.test.graph_lists]
def main(): # check cuda device = f'cuda:{args.gpu}' if args.gpu >= 0 and torch.cuda.is_available( ) else 'cpu' # load ogb dataset & evaluator dataset = DglGraphPropPredDataset(name=args.dataset) evaluator = Evaluator(name=args.dataset) g, _ = dataset[0] node_feat_dim = g.ndata['feat'].size()[-1] edge_feat_dim = g.edata['feat'].size()[-1] n_classes = dataset.num_tasks split_idx = dataset.get_idx_split() train_loader = DataLoader(dataset[split_idx["train"]], batch_size=args.batch_size, shuffle=True, collate_fn=collate_dgl) valid_loader = DataLoader(dataset[split_idx["valid"]], batch_size=args.batch_size, shuffle=False, collate_fn=collate_dgl) test_loader = DataLoader(dataset[split_idx["test"]], batch_size=args.batch_size, shuffle=False, collate_fn=collate_dgl) # load model model = DeeperGCN(dataset=args.dataset, node_feat_dim=node_feat_dim, edge_feat_dim=edge_feat_dim, hid_dim=args.hid_dim, out_dim=n_classes, num_layers=args.num_layers, dropout=args.dropout, learn_beta=args.learn_beta).to(device) print(model) opt = optim.Adam(model.parameters(), lr=args.lr) loss_fn = nn.BCEWithLogitsLoss() # training & validation & testing best_auc = 0 best_model = copy.deepcopy(model) times = [] print('---------- Training ----------') for i in range(args.epochs): t1 = time.time() train_loss = train(model, device, train_loader, opt, loss_fn) t2 = time.time() if i >= 5: times.append(t2 - t1) train_auc = test(model, device, train_loader, evaluator) valid_auc = test(model, device, valid_loader, evaluator) print( f'Epoch {i} | Train Loss: {train_loss:.4f} | Train Auc: {train_auc:.4f} | Valid Auc: {valid_auc:.4f}' ) if valid_auc > best_auc: best_auc = valid_auc best_model = copy.deepcopy(model) print('---------- Testing ----------') test_auc = test(best_model, device, test_loader, evaluator) print(f'Test Auc: {test_auc}') print('Times/epoch: ', sum(times) / len(times))