patience=5, min_lr=0.00001) test_mask = torch.zeros(len(dataset), dtype=torch.uint8) n = len(dataset) // 10 test_mask[i * n:(i + 1) * n] = 1 test_dataset = dataset[test_mask] train_dataset = dataset[1 - test_mask] n = len(train_dataset) // 10 val_mask = torch.zeros(len(train_dataset), dtype=torch.uint8) val_mask[i * n:(i + 1) * n] = 1 val_dataset = train_dataset[val_mask] train_dataset = train_dataset[1 - val_mask] val_loader = DataLoader(val_dataset, batch_size=BATCH) test_loader = DataLoader(test_dataset, batch_size=BATCH) train_loader = DataLoader(train_dataset, batch_size=BATCH, shuffle=True) print('---------------- Split {} ----------------'.format(i)) best_val_loss, test_acc = 100, 0 for epoch in range(1, 101): lr = scheduler.optimizer.param_groups[0]['lr'] train_loss = train(epoch, train_loader, optimizer) val_loss = val(val_loader) scheduler.step(val_loss) if best_val_loss >= val_loss: test_acc = test(test_loader) best_val_loss = val_loss print('Epoch: {:03d}, LR: {:7f}, Train Loss: {:.7f}, '
def main(): # Training settings parser = argparse.ArgumentParser( description='GNN baselines on pcqm4m with Pytorch Geometrics') parser.add_argument('--device', type=int, default=0, help='which gpu to use if any (default: 0)') parser.add_argument( '--gnn', type=str, default='gin-virtual', help= 'GNN gin, gin-virtual, or gcn, or gcn-virtual (default: gin-virtual)') parser.add_argument( '--graph_pooling', type=str, default='sum', help='graph pooling strategy mean or sum (default: sum)') parser.add_argument('--drop_ratio', type=float, default=0, help='dropout ratio (default: 0)') parser.add_argument( '--num_layers', type=int, default=5, help='number of GNN message passing layers (default: 5)') parser.add_argument( '--emb_dim', type=int, default=600, help='dimensionality of hidden units in GNNs (default: 600)') parser.add_argument('--train_subset', action='store_true') parser.add_argument('--batch_size', type=int, default=256, help='input batch size for training (default: 256)') parser.add_argument('--epochs', type=int, default=100, help='number of epochs to train (default: 100)') parser.add_argument('--num_workers', type=int, default=0, help='number of workers (default: 0)') parser.add_argument('--log_dir', type=str, default="", help='tensorboard log directory') parser.add_argument('--use_triplet_loss', action='store_true') parser.add_argument('--checkpoint_dir', type=str, default='', help='directory to save checkpoint') parser.add_argument('--save_test_dir', type=str, default='', help='directory to save test submission file') args = parser.parse_args() print(args) np.random.seed(42) torch.manual_seed(42) torch.cuda.manual_seed(42) random.seed(42) device = torch.device( "cuda:" + str(args.device)) if torch.cuda.is_available() else torch.device("cpu") ### automatic dataloading and splitting dataset = PygPCQM4MDataset(root='dataset/') split_idx = dataset.get_idx_split() ### automatic evaluator. takes dataset name as input evaluator = PCQM4MEvaluator() # if args.use_triplet_loss: # if args.train_subset: # subset_ratio = 0.1 # subset_idx = torch.randperm(len(split_idx["train"]))[:int(subset_ratio*len(split_idx["train"]))] # anchor_loader = DataLoader(dataset[split_idx["train"][subset_idx]], batch_size=args.batch_size, shuffle=True, num_workers = args.num_workers) # positive_loader = DataLoader(dataset[split_idx["train"][subset_idx]], batch_size=args.batch_size, shuffle=True, num_workers = args.num_workers) # negative_loader = DataLoader(dataset[split_idx["train"][subset_idx]], batch_size=args.batch_size, shuffle=True, num_workers = args.num_workers) # else: # anchor_loader = DataLoader(dataset[split_idx["train"]], batch_size=args.batch_size, shuffle=True, num_workers = args.num_workers) # positive_loader = DataLoader(dataset[split_idx["train"]], batch_size=args.batch_size, shuffle=True, num_workers = args.num_workers) # negative_loader = DataLoader(dataset[split_idx["train"]], batch_size=args.batch_size, shuffle=True, num_workers = args.num_workers) # elif args.train_subset: if args.train_subset: subset_ratio = 0.1 subset_idx = torch.randperm(len( split_idx["train"]))[:int(subset_ratio * len(split_idx["train"]))] train_loader = DataLoader(dataset[split_idx["train"][subset_idx]], batch_size=args.batch_size, shuffle=True, num_workers=args.num_workers) else: train_loader = DataLoader(dataset[split_idx["train"]], batch_size=args.batch_size, shuffle=True, num_workers=args.num_workers) valid_loader = DataLoader(dataset[split_idx["valid"]], batch_size=args.batch_size, shuffle=False, num_workers=args.num_workers) if args.save_test_dir is not '': test_loader = DataLoader(dataset[split_idx["test"]], batch_size=args.batch_size, shuffle=False, num_workers=args.num_workers) if args.checkpoint_dir is not '': os.makedirs(args.checkpoint_dir, exist_ok=True) shared_params = { 'num_layers': args.num_layers, 'emb_dim': args.emb_dim, 'drop_ratio': args.drop_ratio, 'graph_pooling': args.graph_pooling } if args.gnn == 'gin': model = GNN(gnn_type='gin', virtual_node=False, **shared_params).to(device) elif args.gnn == 'gin-virtual': model = GNN(gnn_type='gin', virtual_node=True, **shared_params).to(device) elif args.gnn == 'gcn': model = GNN(gnn_type='gcn', virtual_node=False, **shared_params).to(device) elif args.gnn == 'gcn-virtual': model = GNN(gnn_type='gcn', virtual_node=True, **shared_params).to(device) elif args.gnn == 'gin-virtual-bnn': model = BayesianGNN(gnn_type='gin', virtual_node=True, last_layer_only=False, **shared_params).to(device) elif args.gnn == 'gin-virtual-bnn-lastLayer': model = BayesianGNN(gnn_type='gin', virtual_node=True, last_layer_only=True, **shared_params).to(device) else: raise ValueError('Invalid GNN type') num_params = sum(p.numel() for p in model.parameters()) print(f'#Params: {num_params}') optimizer = optim.Adam(model.parameters(), lr=0.001) if args.log_dir is not '': writer = SummaryWriter(log_dir=args.log_dir) best_valid_mae = 1000 if args.train_subset: scheduler = StepLR(optimizer, step_size=300, gamma=0.25) args.epochs = 1000 else: scheduler = StepLR(optimizer, step_size=30, gamma=0.25) # start epoch (default = 1, unless resuming training) firstEpoch = 1 # check if checkpoint exist -> load model checkpointFile = os.path.join(args.checkpoint_dir, 'checkpoint.pt') if os.path.exists(checkpointFile): # load checkpoint file checkpointData = torch.load(checkpointFile) firstEpoch = checkpointData["epoch"] model.load_state_dict(checkpointData["model_state_dict"]) optimizer.load_state_dict(checkpointData["optimizer_state_dict"]) scheduler.load_state_dict(checkpointData["scheduler_state_dict"]) best_valid_mae = checkpointData["best_val_mae"] num_params = checkpointData["num_params"] print( "Loaded existing weights from {}. Continuing from epoch: {} with best valid MAE: {}" .format(checkpointFile, firstEpoch, best_valid_mae)) if args.use_triplet_loss: model.gnn_node.register_forward_hook(get_activation('gnn_node')) for epoch in range(firstEpoch, args.epochs + 1): print("=====Epoch {}".format(epoch)) print('Training...') if args.use_triplet_loss: train_mae = triplet_loss_train(model, device, train_loader, dataset, optimizer, args.gnn, args) else: train_mae = train(model, device, train_loader, optimizer, args.gnn) print('Evaluating...') valid_mae = eval(model, device, valid_loader, evaluator) print({'Train': train_mae, 'Validation': valid_mae}) if args.log_dir is not '': writer.add_scalar('valid/mae', valid_mae, epoch) writer.add_scalar('train/mae', train_mae, epoch) if valid_mae < best_valid_mae: best_valid_mae = valid_mae if args.checkpoint_dir is not '': print('Saving checkpoint...') checkpoint = { 'epoch': epoch, 'model_state_dict': model.state_dict(), 'optimizer_state_dict': optimizer.state_dict(), 'scheduler_state_dict': scheduler.state_dict(), 'best_val_mae': best_valid_mae, 'num_params': num_params } torch.save(checkpoint, os.path.join(args.checkpoint_dir, 'checkpoint.pt')) if args.save_test_dir is not '': print('Predicting on test data...') y_pred = test(model, device, test_loader) print('Saving test submission file...') evaluator.save_test_submission({'y_pred': y_pred}, args.save_test_dir) scheduler.step() print(f'Best validation MAE so far: {best_valid_mae}') if args.log_dir is not '': writer.close()
import os.path as osp import torch import time import torch.nn.functional as F from torch.nn import Sequential, Linear, ReLU from torch_geometric.datasets import TUDataset from torch_geometric.data import DataLoader from torch_geometric.nn import GINConv, global_add_pool path = osp.join(osp.dirname(osp.realpath(__file__)), '..', 'data', 'NCI1') dataset = TUDataset(path, name='NCI1').shuffle() test_dataset = dataset[:len(dataset) // 10] train_dataset = dataset[len(dataset) // 10:] test_loader = DataLoader(test_dataset, batch_size=256) train_loader = DataLoader(train_dataset, batch_size=256) class Net(torch.nn.Module): def __init__(self): super(Net, self).__init__() num_features = dataset.num_features dim = 32 nn1 = Sequential(Linear(num_features, dim), ReLU(), Linear(dim, dim)) self.conv1 = GINConv(nn1) self.bn1 = torch.nn.BatchNorm1d(dim) nn2 = Sequential(Linear(dim, dim), ReLU(), Linear(dim, dim)) self.conv2 = GINConv(nn2)
return label def if_same_col(self, si, ti, tbpos): ss, se = tbpos[si][2], tbpos[si][3] ts, te = tbpos[ti][2], tbpos[ti][3] if (ss >= ts and se <= te): return 2 if (ts >= ss and te <= se): return 2 return 0 def if_same_cell(self): pass if __name__ == "__main__": root_path = '/home/jiuxiao/NLP/table/data/SciTSR/train' ds = ScitsrDataset(root_path) print(len(ds)) # ds.check_all() # ds.get(0) #print(ds.get_html(76)) test_loader = DataLoader(ds, batch_size=5) for data in test_loader: print(data, data.num_graphs) x = scatter_mean(data.x, data.batch, dim=0) print(data.edge_index) print(x.size()) print("ratio:", data.y.sum().float() / data.y.size()[0])
def train_loader(self): return DataLoader(self._train_data, shuffle=self._shuffle, batch_size=self._batch_size)
print('- weight decay: ',weight_decay) print('- hidden size: ',nhid) #print('- graph convolution: ','GCNConv') print('- number of graph convoluational layers: {}x{}'.format(2,num_layers)) #print('- graph pooling: ','HaarPooling') print('- number of pooling layers: ',num_layers) print('- number of fully connected layers: ',num_layers) print('- maximum number of epochs: ',epochs) print('- patience for earlystopping: ',early_stopping) print('Datasets') #print('- name: ',dataname) print('- number of training data:',num_training) print('- number of validation data:',num_val) print('- number of test data:',num_test) train_loader = DataLoader(training_set, batch_size=batch_size, shuffle = False) val_loader = DataLoader(validation_set, batch_size=batch_size, shuffle = False) # add validation for a possible early stopping test_loader = DataLoader(test_set, batch_size=batch_size, shuffle = False) #%% ##Define Model Class class Net(torch.nn.Module): def __init__(self): super(Net, self).__init__() self.nhid = nhid self.num_layers = num_layers self.conv1 = GCNConv(num_features, self.nhid) self.conv2 = GCNConv(self.nhid, self.nhid) self.conv3 = GCNConv(self.nhid, self.nhid) self.conv4 = GCNConv(self.nhid, self.nhid)
def val_dataloader(self): return DataLoader(self.test_dataset, batch_size=64)
print('Feature matrix: ', example.x) print('Edge features: ', example.edge_attr) print('Label: ', example.y) print() # preprocess graph data graph_data = normalizeFeatures(graph_data) # create train, val test split num_training = int(len(graph_data) * 0.6) num_val = int(len(graph_data) * 0.1) num_test = len(graph_data) - (num_training + num_val) training_set, validation_set, test_set = random_split(graph_data, [num_training, num_val, num_test]) # create dataloader objects train_loader = DataLoader(training_set, batch_size=args.batch_size, shuffle=True) val_loader = DataLoader(validation_set, batch_size=args.batch_size, shuffle=False) test_loader = DataLoader(test_set, batch_size=args.batch_size, shuffle=False) # Create the model class Net(torch.nn.Module): def __init__(self): super(Net, self).__init__() self.num_features = args.num_features self.nhid = args.nhid self.conv1 = GCNConv(self.num_features, self.nhid * 2) self.conv2 = GCNConv(self.nhid * 2, self.nhid * 2)
def cross_validation_with_val_set(dataset, model, folds, epochs, batch_size, lr, lr_decay_factor, lr_decay_step_size, weight_decay, logger=None): val_losses, accs, durations = [], [], [] for fold, (train_idx, test_idx, val_idx) in enumerate(zip(*k_fold(dataset, folds))): train_dataset = dataset[train_idx] test_dataset = dataset[test_idx] val_dataset = dataset[val_idx] if 'adj' in train_dataset[0]: train_loader = DenseLoader(train_dataset, batch_size, shuffle=True) val_loader = DenseLoader(val_dataset, batch_size, shuffle=False) test_loader = DenseLoader(test_dataset, batch_size, shuffle=False) else: train_loader = DataLoader(train_dataset, batch_size, shuffle=True) val_loader = DataLoader(val_dataset, batch_size, shuffle=False) test_loader = DataLoader(test_dataset, batch_size, shuffle=False) model.to(device).reset_parameters() optimizer = Adam(model.parameters(), lr=lr, weight_decay=weight_decay) if torch.cuda.is_available(): torch.cuda.synchronize() t_start = time.perf_counter() for epoch in range(1, epochs + 1): train_loss = train(model, optimizer, train_loader) val_losses.append(eval_loss(model, val_loader)) accs.append(eval_acc(model, test_loader)) eval_info = { 'fold': fold, 'epoch': epoch, 'train_loss': train_loss, 'val_loss': val_losses[-1], 'test_acc': accs[-1], } if logger is not None: logger(eval_info) if epoch % lr_decay_step_size == 0: for param_group in optimizer.param_groups: param_group['lr'] = lr_decay_factor * param_group['lr'] if torch.cuda.is_available(): torch.cuda.synchronize() t_end = time.perf_counter() durations.append(t_end - t_start) loss, acc, duration = tensor(val_losses), tensor(accs), tensor(durations) loss, acc = loss.view(folds, epochs), acc.view(folds, epochs) loss, argmin = loss.min(dim=1) acc = acc[torch.arange(folds, dtype=torch.long), argmin] print('Val Loss: {:.4f}, Test Accuracy: {:.3f} ± {:.3f}, Duration: {:.3f}'. format(loss.mean().item(), acc.mean().item(), acc.std().item(), duration.mean().item())) return loss.mean().item(), acc.mean().item(), acc.std().item()
def atexit_tasks(model): # ----------------------------------------------- # ---------------- EVALUATION ROUTINE ----------- # ----------------------------------------------- # save the tensorboardx summary files summary_dir_exit = os.path.join( config.run_abs_path, config.summary_dir) summary_compressed = summary_dir_exit + '.tar.gz' # remove old tar file if os.path.isfile(summary_compressed): os.remove(summary_compressed) with tarfile.open(summary_compressed, mode='w:gz') as archive: archive.add(summary_dir_exit, arcname='summary', recursive=True) _run.add_artifact(filename=summary_compressed, name='summary.tar.gz') model.eval() model.current_writer = None # train loss final_loss_train = 0.0 final_metric_train = 0.0 final_nr_nodes_train = 0 for data_ft in data_loader_train: data_ft = data_ft.to(device) out_ft = model(data_ft) final_loss_train += model.loss(out_ft, data_ft.y).item() * data_ft.num_nodes final_metric_train += model.out_to_metric( out_ft, data_ft.y) * data_ft.num_nodes final_nr_nodes_train += data_ft.num_nodes final_loss_train /= final_nr_nodes_train final_metric_train /= final_nr_nodes_train _run.log_scalar( 'loss_train_final', final_loss_train, config.training_epochs) _run.log_scalar( 'accuracy_train_final', final_metric_train, config.training_epochs) # test loss data_loader_test = DataLoader( test_dataset, batch_size=config.batch_size_eval, shuffle=False) test_loss = 0.0 test_metric = 0.0 nr_nodes_test = 0 test_predictions = [] test_targets = [] for data_fe in data_loader_test: data_fe = data_fe.to(device) out_fe = model(data_fe) test_loss += model.loss(out_fe, data_fe.y).item() * data_fe.num_nodes test_metric += model.out_to_metric(out_fe, data_fe.y) * data_fe.num_nodes nr_nodes_test += data_fe.num_nodes pred = model.out_to_predictions(out_fe) test_predictions.extend(model.predictions_to_list(pred)) test_targets.extend(data_fe.y.tolist()) test_loss /= nr_nodes_test test_metric /= nr_nodes_test _run.log_scalar('loss_test', test_loss, config.training_epochs) _run.log_scalar('accuracy_test', test_metric, config.training_epochs) # final print routine print('') dataset.print_summary() print('Total number of parameters: {}'.format(total_params)) print('Mean train loss ({0} samples): {1:.3f}'.format( train_dataset.__len__(), final_loss_train)) print('Mean accuracy on train set: {0:.3f}'.format( final_metric_train)) print('Mean test loss ({0} samples): {1:.3f}'.format( test_dataset.__len__(), test_loss)) print('Mean accuracy on test set: {0:.3f}'.format( test_metric)) print('') # plot targets vs predictions. default is a confusion matrix model.plot_targets_vs_predictions( targets=test_targets, predictions=test_predictions) _run.add_artifact( filename=os.path.join( config.run_abs_path, config.confusion_matrix_path), name=config.confusion_matrix_path) # if Regression, plot targets vs. continuous outputs # if isinstance(model.model_type, RegressionProblem): # test_outputs = [] # for data in data_loader_test: # data = data.to(device) # out = torch.squeeze(model(data)).tolist() # test_outputs.extend(out) # model.model_type.plot_targets_vs_outputs( # targets=test_targets, outputs=test_outputs) # plot errors by location # plotter = ResultPlotting(config=config) # plotter.plot_errors_by_location( # data=test_dataset, predictions=test_predictions, # targets=test_targets) # plot the graphs in the test dataset for visual inspection if config.plot_graphs_testset: if config.plot_graphs_testset < 0 or config.plot_graphs_testset > test_dataset.__len__(): plot_limit = test_dataset.__len__() else: plot_limit = config.plot_graphs_testset for i in range(plot_limit): g = test_dataset[i] g.to(device) out_p = model(g) g.plot_predictions( config=config, pred=model.predictions_to_list( model.out_to_predictions(out_p)), graph_nr=i, run=_run, acc=model.out_to_metric( out_p, g.y), logger=_log) return '\n{0}\ntrain acc: {1:.3f}\ntest acc: {2:.3f}'.format( _run.meta_info['options']['--comment'], final_metric_train, test_metric)
def main(_config, _run, _log): # Check for a comment, if none is given raise error if _run.meta_info['options']['--comment'] is None: raise ValueError('You need to specify a comment with -c, --comment') config = argparse.Namespace(**_config) _log.info('Logging to {}'.format(config.run_abs_path)) # ----------------------------------------------- # ---------------- CREATE SETUP ----------------- # ----------------------------------------------- # make necessary directory structure if not os.path.isdir(config.run_abs_path): os.makedirs(config.run_abs_path) # clear old stuff from the run dir, if it's not a restart if not config.load_model: summary_dir = os.path.join(config.run_abs_path, config.summary_dir) if os.path.isdir(summary_dir): shutil.rmtree(summary_dir) model_dir = os.path.join(config.run_abs_path, config.model_dir) if os.path.isdir(model_dir): shutil.rmtree(model_dir) # make dir structure in temp dir os.makedirs(summary_dir) os.makedirs(model_dir) # Pass the path of tensorboardX summaries to sacred if config.write_summary: _run.info["tensorflow"] = dict() _run.info["tensorflow"]["logdirs"] = [os.path.join( config.run_abs_path, config.summary_dir)] # set up the summary writer for tensorboardX train_writer = SummaryWriter(os.path.join( config.run_abs_path, 'summary', 'training')) val_writer = SummaryWriter(os.path.join( config.run_abs_path, 'summary', 'validation')) # create and load dataset dataset = globals()[config.dataset_type]( root=config.dataset_abs_path, config=config) dataset.update_config(config) assert dataset[0].edge_attr.size(1) == config.pseudo_dimensionality if config.standardize_targets: config.targets_mean, config.targets_std = dataset.targets_mean_std() # TODO if model is loaded, use the same train val test split. # shuffle can return the permutation of the dataset, which can then be used to permute the same way # dataset, perm = dataset.shuffle(return_perm=True) # when loading a model: # dataset = dataset.__indexing__(permutation) # split into train and test split_train_idx = int( config.samples * (1 - config.test_split - config.validation_split)) split_validation_idx = int(config.samples * (1 - config.test_split)) train_dataset = dataset[:split_train_idx] validation_dataset = dataset[split_train_idx:split_validation_idx] test_dataset = dataset[split_validation_idx:] device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') data_loader_train = DataLoader( train_dataset, batch_size=config.batch_size_train, shuffle=True) data_loader_validation = DataLoader( validation_dataset, batch_size=config.batch_size_eval, shuffle=False) if not config.load_model: model = globals()[config.model]( config=config, train_writer=train_writer, val_writer=val_writer, model_type=config.model_type ) model = model.to(device) else: _log.info('Loading model {} ...'.format(config.load_model)) # TODO allow to load previous models # find latest state of model load_model_dir = os.path.join( config.root_dir, config.run_abs_path, config.model_dir) checkpoint_versions = [name for name in os.listdir( load_model_dir) if name.endswith('.tar')] if 'final.tar' in checkpoint_versions: checkpoint_to_load = 'final.tar' else: checkpoint_versions = [ x for x in checkpoint_versions if x.startswith('epoch')].sort() checkpoint_to_load = checkpoint_versions[-1] _log.info('Loading checkpoint {} ...'.format( os.path.join(load_model_dir, checkpoint_to_load))) checkpoint = torch.load(os.path.join( load_model_dir, checkpoint_to_load)) # restore the checkpoint model = globals()[config.model]( config=config, train_writer=train_writer, val_writer=val_writer, epoch=checkpoint['epoch'], train_batch_iteration=checkpoint['train_batch_iteration'], val_batch_iteration=checkpoint['val_batch_iteration'], model_type=config.model_type ) # model.to(device) has to be executed before loading the state # dicts model.to(device) model.load_state_dict(checkpoint['model_state_dict']) model.optimizer.load_state_dict(checkpoint['optimizer_state_dict']) total_params = sum(p.numel() for p in model.parameters() if p.requires_grad) _run.log_scalar('nr_params', total_params, config.training_epochs) # save config to file and store in DB config_filepath = os.path.join(config.run_abs_path, 'config.json') with open(config_filepath, 'w') as f: json.dump(vars(config), f) _run.add_artifact(filename=config_filepath) def atexit_tasks(model): # ----------------------------------------------- # ---------------- EVALUATION ROUTINE ----------- # ----------------------------------------------- # save the tensorboardx summary files summary_dir_exit = os.path.join( config.run_abs_path, config.summary_dir) summary_compressed = summary_dir_exit + '.tar.gz' # remove old tar file if os.path.isfile(summary_compressed): os.remove(summary_compressed) with tarfile.open(summary_compressed, mode='w:gz') as archive: archive.add(summary_dir_exit, arcname='summary', recursive=True) _run.add_artifact(filename=summary_compressed, name='summary.tar.gz') model.eval() model.current_writer = None # train loss final_loss_train = 0.0 final_metric_train = 0.0 final_nr_nodes_train = 0 for data_ft in data_loader_train: data_ft = data_ft.to(device) out_ft = model(data_ft) final_loss_train += model.loss(out_ft, data_ft.y).item() * data_ft.num_nodes final_metric_train += model.out_to_metric( out_ft, data_ft.y) * data_ft.num_nodes final_nr_nodes_train += data_ft.num_nodes final_loss_train /= final_nr_nodes_train final_metric_train /= final_nr_nodes_train _run.log_scalar( 'loss_train_final', final_loss_train, config.training_epochs) _run.log_scalar( 'accuracy_train_final', final_metric_train, config.training_epochs) # test loss data_loader_test = DataLoader( test_dataset, batch_size=config.batch_size_eval, shuffle=False) test_loss = 0.0 test_metric = 0.0 nr_nodes_test = 0 test_predictions = [] test_targets = [] for data_fe in data_loader_test: data_fe = data_fe.to(device) out_fe = model(data_fe) test_loss += model.loss(out_fe, data_fe.y).item() * data_fe.num_nodes test_metric += model.out_to_metric(out_fe, data_fe.y) * data_fe.num_nodes nr_nodes_test += data_fe.num_nodes pred = model.out_to_predictions(out_fe) test_predictions.extend(model.predictions_to_list(pred)) test_targets.extend(data_fe.y.tolist()) test_loss /= nr_nodes_test test_metric /= nr_nodes_test _run.log_scalar('loss_test', test_loss, config.training_epochs) _run.log_scalar('accuracy_test', test_metric, config.training_epochs) # final print routine print('') dataset.print_summary() print('Total number of parameters: {}'.format(total_params)) print('Mean train loss ({0} samples): {1:.3f}'.format( train_dataset.__len__(), final_loss_train)) print('Mean accuracy on train set: {0:.3f}'.format( final_metric_train)) print('Mean test loss ({0} samples): {1:.3f}'.format( test_dataset.__len__(), test_loss)) print('Mean accuracy on test set: {0:.3f}'.format( test_metric)) print('') # plot targets vs predictions. default is a confusion matrix model.plot_targets_vs_predictions( targets=test_targets, predictions=test_predictions) _run.add_artifact( filename=os.path.join( config.run_abs_path, config.confusion_matrix_path), name=config.confusion_matrix_path) # if Regression, plot targets vs. continuous outputs # if isinstance(model.model_type, RegressionProblem): # test_outputs = [] # for data in data_loader_test: # data = data.to(device) # out = torch.squeeze(model(data)).tolist() # test_outputs.extend(out) # model.model_type.plot_targets_vs_outputs( # targets=test_targets, outputs=test_outputs) # plot errors by location # plotter = ResultPlotting(config=config) # plotter.plot_errors_by_location( # data=test_dataset, predictions=test_predictions, # targets=test_targets) # plot the graphs in the test dataset for visual inspection if config.plot_graphs_testset: if config.plot_graphs_testset < 0 or config.plot_graphs_testset > test_dataset.__len__(): plot_limit = test_dataset.__len__() else: plot_limit = config.plot_graphs_testset for i in range(plot_limit): g = test_dataset[i] g.to(device) out_p = model(g) g.plot_predictions( config=config, pred=model.predictions_to_list( model.out_to_predictions(out_p)), graph_nr=i, run=_run, acc=model.out_to_metric( out_p, g.y), logger=_log) return '\n{0}\ntrain acc: {1:.3f}\ntest acc: {2:.3f}'.format( _run.meta_info['options']['--comment'], final_metric_train, test_metric) atexit.register(atexit_tasks, model=model) # ----------------------------------------------- # ---------------- TRAINING LOOP ---------------- # ----------------------------------------------- for epoch in range(model.epoch, config.training_epochs): # put model in training mode (e.g. use dropout) model.train() epoch_loss = 0.0 epoch_metric_train = 0.0 nr_nodes_train = 0 _log.info('epoch {} ...'.format(epoch)) for batch_i, data in enumerate(data_loader_train): data = data.to(device) # call the forward method out = model(data) loss = model.loss(out, data.y) model.print_current_loss(epoch, batch_i, _log) epoch_loss += loss.item() * data.num_nodes epoch_metric_train += model.out_to_metric( out, data.y) * data.num_nodes nr_nodes_train += data.num_nodes # clear the gradient variables of the model model.optimizer.zero_grad() loss.backward() # Gradient clipping if config.clip_grad: if config.clip_method == 'value': torch.nn.utils.clip_grad_value_( parameters=model.parameters(), clip_value=config.clip_value ) else: torch.nn.utils.clip_grad_norm_( parameters=model.parameters(), max_norm=config.clip_value, norm_type=float(config.clip_method) ) model.optimizer.step() model.train_batch_iteration += 1 epoch_loss /= nr_nodes_train epoch_metric_train /= nr_nodes_train if config.write_summary: train_writer.add_scalar('_per_epoch/loss', epoch_loss, epoch) train_writer.add_scalar( '_per_epoch/metric', epoch_metric_train, epoch) _run.log_scalar('loss_train', epoch_loss, epoch) _run.log_scalar('accuracy_train', epoch_metric_train, epoch) # validation model.eval() validation_loss = 0.0 epoch_metric_val = 0.0 nr_nodes_val = 0 for batch_i, data in enumerate(data_loader_validation): data = data.to(device) out = model(data) loss = model.loss(out, data.y) model.print_current_loss( epoch, 'validation {}'.format(batch_i), _log) validation_loss += loss.item() * data.num_nodes epoch_metric_val += model.out_to_metric( out, data.y) * data.num_nodes nr_nodes_val += data.num_nodes model.val_batch_iteration += 1 # The numbering of train and val does not correspond 1-to-1! # Here we skip some numbers for maintaining loose correspondence model.val_batch_iteration = model.train_batch_iteration validation_loss /= nr_nodes_val epoch_metric_val /= nr_nodes_val if config.write_summary: val_writer.add_scalar('_per_epoch/loss', validation_loss, epoch) val_writer.add_scalar('_per_epoch/metric', epoch_metric_val, epoch) _run.log_scalar('loss_val', validation_loss, epoch) _run.log_scalar('accuracy_val', epoch_metric_val, epoch) model.epoch += 1 # save intermediate models if model.epoch % config.checkpoint_interval == 0: model.save('epoch_{}'.format(model.epoch)) # save the final model final_model_name = 'final' model.save(final_model_name) _run.add_artifact( filename=os.path.join( config.run_abs_path, config.model_dir, final_model_name + '.tar'), name=final_model_name) ########################### return atexit_tasks(model=model)
model.load_state_dict(state['state_dict']) optimizer.load_state_dict(state['optimizer']) print('model loaded from %s' % checkpoint_path) #%% if __name__ == "__main__": np.random.seed(SEED) torch.manual_seed(SEED) # hyper parameters train_data = GraphDataset(TRAIN_DIR).shuffle() val_data = GraphDataset(VAL_DIR) if small_dataset: train_loader = DataLoader(train_data[:1000], batch_size=batch_size, shuffle=True) val_loader = DataLoader(val_data[:200], batch_size=batch_size) else: train_loader = DataLoader(train_data, batch_size=batch_size, shuffle=True) val_loader = DataLoader(val_data, batch_size=batch_size) model = HGNN(in_channels, out_channels).to(device) optimizer = optim.Adam(model.parameters(), lr=lr) scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=decay_lr_every, gamma=decay_lr_factor) if checkpoint_dir: load_checkpoint(checkpoint_dir, model, optimizer)
num_nodes = [len(M[i].v) for i in range(len(M))] print('Loading Dataset') if args.data_dir: data_dir = args.data_dir else: data_dir = config['data_dir'] normalize_transform = Normalize() dataset = ComaDataset(data_dir, dtype='test', split=args.split, split_term=args.split_term, pre_transform=normalize_transform) data_loader = DataLoader(dataset, batch_size=1, shuffle=True, num_workers=1) print('Loading model') coma = Coma(dataset, config, D_t, U_t, A_t, num_nodes) checkpoint_file = config['checkpoint_file'] print(checkpoint_file) if checkpoint_file: checkpoint = torch.load(checkpoint_file, map_location=torch.device('cpu')) coma.load_state_dict(checkpoint['state_dict']) coma.to(device) meshviewer = MeshViewers(shape=(3, cols)) for row in meshviewer:
P2 = [0.01] H = torch.from_numpy(error_generate.generate_PCM(2 * L * L - 2, L)).t() #64, 30 h_prep = error_generate.H_Prep(H.t()) H_prep = torch.from_numpy(h_prep.get_H_Prep()) BATCH_SIZE = 128 lr = 3e-4 Nc = 10 run1 = 40960 run2 = 8192 dataset1 = error_generate.gen_syn(P1, L, H, run1) dataset2 = error_generate.gen_syn(P2, L, H, run2) train_dataset = CustomDataset(H, dataset1) test_dataset = CustomDataset(H, dataset2) rows, cols = H.size(0), H.size(1) train_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=False) test_loader = DataLoader(test_dataset, batch_size=BATCH_SIZE, shuffle=False) logical, stab = h_prep.get_logical(H_prep) logical, stab = logical.cuda(), stab.cuda() def init_weights(m): if type(m) == torch.nn.Linear: # torch.nn.init.xavier_uniform_(m.weight) # torch.nn.init.constant_(m.weight, val) # torch.nn.init.uniform_(m.weight, a=0.0884, b=0.1) torch.nn.init.uniform_(m.weight, a=0.1, b=1) m.bias.data.fill_(1e-3) def init_weights_one(m):
import os.path as osp import torch import torch.nn.functional as F from torch_geometric.datasets import PPI from torch_geometric.data import DataLoader from torch_geometric.nn import GATConv from sklearn import metrics path = osp.join(osp.dirname(osp.realpath(__file__)), '..', 'data', 'PPI') train_dataset = PPI(path, split='train') val_dataset = PPI(path, split='test') test_dataset = PPI(path, split='test') train_loader = DataLoader(train_dataset, batch_size=1, shuffle=True) val_loader = DataLoader(val_dataset, batch_size=2, shuffle=False) test_loader = DataLoader(test_dataset, batch_size=2, shuffle=False) dataset_name = 'PPI' class Net(torch.nn.Module): def __init__(self): super(Net, self).__init__() self.conv1 = GATConv(train_dataset.num_features, 256, heads=4) self.lin1 = torch.nn.Linear(train_dataset.num_features, 4 * 256) self.conv2 = GATConv(4 * 256, 256, heads=4) self.lin2 = torch.nn.Linear(4 * 256, 4 * 256) self.conv3 = GATConv(4 * 256, train_dataset.num_classes, heads=6, concat=False) self.lin3 = torch.nn.Linear(4 * 256, train_dataset.num_classes)
path_graph_input = os.path.join(path_data_root, 'input_graph') dataset_atac_graph = ATACGraphDataset(path_graph_input) torch.manual_seed(12345) dataset = dataset_atac_graph.shuffle() train_dataset = dataset[:1700] test_dataset = dataset[1700:] device = torch.device("cuda:1" if torch.cuda.is_available() else "cpu") model = GCN(input_channels=dataset.num_node_features, output_channels=dataset.num_classes, hidden_channels=8, num_nodes=dataset_atac_graph[0].num_nodes).to(device) criterion = torch.nn.CrossEntropyLoss() train_loader = DataLoader(train_dataset, batch_size=32, shuffle=True) test_loader = DataLoader(test_dataset, batch_size=32, shuffle=False) # train model time_start = time() optimizer = torch.optim.Adam(model.parameters(), lr=0.0005) for epoch in range(1, 50): train(train_loader) train_acc = test(train_loader) test_acc = test(test_loader) print(f'Epoch: {epoch:03d}, ' f'Train Acc: {train_acc:.4f}, Test Acc: {test_acc:.4f}') if test_acc > 0.97: break time_end = time() print(time_end - time_start)
super(PyGToyDataset, self).__init__(save_root, transform, pre_transform) self.data, self.slices = torch.load(self.processed_file_names[0]) @property def raw_file_names(self): return ["origin_dataset"] @property def processed_file_names(self): return ["toy_dataset.pt"] def download(self): pass def process(self): # 100 samples, each sample is a graph with 32 nodes and 42 edges, each node has a feature dimension of 3. data_list = [toy_dataset(num_nodes=32, num_node_features=3, num_edges=42) for _ in range(100)] data_save, data_slices = self.collate(data_list) torch.save((data_save, data_slices), self.processed_file_names[0]) if __name__ == '__main__': # toy_sample = toy_dataset(num_nodes=32, num_node_features=3, num_edges=42) # print(toy_sample) toy_data = PyGToyDataset(save_root="toy") # 100 samples, each sample is a graph # print(toy_data[0]) data_loader = DataLoader(toy_data, batch_size=5, shuffle=True) for batch in data_loader: print(batch)
optimizer = torch.optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.weight_decay) earlystop = Earlystop(window_size=50) train_dataset = [] test_dataset = [] for j in range(10): if j == fold: test_dataset = folded_data[j] else: train_dataset.extend(folded_data[j]) valid_idx = int(len(train_dataset) * 0.1) valid_dataset = train_dataset[:valid_idx] train_dataset = train_dataset[valid_idx:] test_loader = DataLoader(test_dataset, batch_size=args.batch_size) valid_loader = DataLoader(valid_dataset, batch_size=args.batch_size) train_loader = DataLoader(train_dataset, batch_size=args.batch_size) fold_best_acc = 0. fold_val_loss = 100. fold_val_acc = 0. patience = 0 for epoch in range(1, 100001): t = time.time() loss = train(train_loader) val_acc, val_loss = test(valid_loader) train_acc, _ = test(train_loader) if fold_val_loss >= val_loss:
def train_dataloader(self): return DataLoader(self.train_dataset, batch_size=64, shuffle=True)
def modelSelection(model_list,k, train_dataset, balanced=True, force_numclasses=None, unbalanced_split=False ): global device device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') #print(dir(train_dataset)) # not working if force_numclasses is not None: train_dataset.num_classes = force_numclasses kfolds = kFolding2(train_dataset,k, balanced, unbalanced_split) for modeldict in model_list: train_loss_history = [] val_history = {'loss':[], 'accuracy':[], 'microF1':[],'macroF1':[]} modeldict['cv_val_loss']=0.0 modeldict['cv_val_accuracy']=0.0 modeldict['cv_val_microF1'] =0.0 modeldict['cv_val_macroF1'] =0.0 epochs = modeldict['epochs'] modelclass = modeldict['model'] kwargs = modeldict['kwargs'] try: model = modelclass(**kwargs) model = model.to(device) modeldict['model_instance'] = model lr = modeldict['learning_rate'] wd = modeldict['weight_decay'] bs = modeldict['batch_size'] optimizer = torch.optim.Adam(model.parameters(), lr=lr, weight_decay=wd) for kfold in kfolds: train = train_dataset[kfold[0]] val = train_dataset[kfold[1]] loader = DataLoader(train, batch_size=bs, shuffle=True) loader_val = DataLoader(val, batch_size=bs, shuffle=True) for epoch in range(epochs): train_model(model, loader, optimizer, train_loss_history) val_loss_model(model, loader_val, optimizer, val_history) # save results modeldict['train_loss_history']=train_loss_history modeldict['val_loss_history']=val_history['loss'] modeldict['val_accuracy_history']=val_history['accuracy'] modeldict['val_loss']=val_history['loss'][-1] modeldict['accuracy']=val_history['accuracy'][-1] modeldict['microF1']=val_history['microF1'][-1] modeldict['macroF1']=val_history['macroF1'][-1] modeldict['cv_val_loss']+=modeldict['val_loss'] modeldict['cv_val_accuracy']+=modeldict['accuracy'] modeldict['cv_val_microF1']+=modeldict['microF1'] modeldict['cv_val_macroF1']+=modeldict['macroF1'] modeldict['cv_val_loss']=modeldict['cv_val_loss']/len(kfolds) modeldict['cv_val_accuracy']=modeldict['cv_val_accuracy']/len(kfolds) modeldict['cv_val_microF1']=modeldict['cv_val_microF1']/len(kfolds) modeldict['cv_val_macroF1']=modeldict['cv_val_macroF1']/len(kfolds) except: print("Problem training model "+modeldict['model'].__name__) traceback.print_exc() # report model results reportTrainedModel(modeldict) # select best model modelsdict = selectBestModel(model_list) # save model to disk + save file path # or save model in the dict.. (could take too much memory) saveModels(modelsdict) return modelsdict
import os.path as osp import torch import torch.nn.functional as F from torch_geometric.datasets import MNISTSuperpixels import torch_geometric.transforms as T from torch_geometric.data import DataLoader from torch_geometric.nn import SplineConv, voxel_grid, max_pool, max_pool_x path = osp.join(osp.dirname(osp.realpath(__file__)), '..', 'data', 'MNIST') transform = T.Cartesian(cat=False) train_dataset = MNISTSuperpixels(path, True, transform=transform) test_dataset = MNISTSuperpixels(path, False, transform=transform) train_loader = DataLoader(train_dataset, batch_size=64, shuffle=True) test_loader = DataLoader(test_dataset, batch_size=64) d = train_dataset class Net(torch.nn.Module): def __init__(self): super(Net, self).__init__() self.conv1 = SplineConv(d.num_features, 32, dim=2, kernel_size=5) self.conv2 = SplineConv(32, 64, dim=2, kernel_size=5) self.conv3 = SplineConv(64, 64, dim=2, kernel_size=5) self.fc1 = torch.nn.Linear(4 * 64, 128) self.fc2 = torch.nn.Linear(128, d.num_classes) def forward(self, data): data.x = F.elu(self.conv1(data.x, data.edge_index, data.edge_attr)) cluster = voxel_grid(data.pos, data.batch, size=5, start=0, end=28) data.edge_attr = None
# Normalize targets to mean = 0 and std = 1. mean = dataset.data.y[:, target].mean().item() std = dataset.data.y[:, target].std().item() dataset.data.y[:, target] = (dataset.data.y[:, target] - mean) / std # print(type(dataset[0])) # print(type(dataset.data.x)) #tensor # print(type(dataset.data.y)) #tensor # Split datasets. test_dataset = dataset[:10000] val_dataset = dataset[10000:20000] train_dataset = dataset[20000:20000 + args.train_num] test_loader = DataLoader(test_dataset, batch_size=batch_size) val_loader = DataLoader(val_dataset, batch_size=batch_size) train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True) if use_unsup_loss: unsup_train_dataset = dataset[20000:] unsup_train_loader = DataLoader(unsup_train_dataset, batch_size=batch_size, shuffle=True) print(len(train_dataset), len(val_dataset), len(test_dataset), len(unsup_train_dataset)) else: print(len(train_dataset), len(val_dataset), len(test_dataset))
from utils.CustomDataSet import SelectGraph, SceneGraphs from classification.Graph_AE import Net from classification.Classifier import MLP import utils.Display_Plot as dp import torch device = torch.device('cuda:1') num_epoch = 100 batch_size = 200 comp_model = Net.get_instance().to(device) cfy_model = MLP.get_instance().to(device) SelectGraph.data_name = 'Shana10k' data_set_Shana = SelectGraph('data/' + SelectGraph.data_name) train_set = DataLoader(data_set_Shana[:5000], 500, shuffle=True) SelectGraph.data_name = 'Shana7000' data_set_Shana = SelectGraph('data/' + SelectGraph.data_name) train_set2 = DataLoader(data_set_Shana[5000:6000], 1000, shuffle=False) test_set = DataLoader(data_set_Shana[6000:7000], 1000, shuffle=False) m_name = "GAE_TRANSFER.ckpt" data_list1, group1, group2 = comp_model.train_model(train_set, train_set2, test_set, num_epoch, m_name) data_list2 = cfy_model.train_model(train_set2, test_set, int(num_epoch), group1, group2) title = "GAE TRANSFER" labels = ['MSE Loss', 'Num Nodes', 'Total Loss', title]
import torch import torch.nn.functional as F from torch.nn import Sequential as Seq, Dropout, Linear as Lin from torch_geometric.datasets import ModelNet import torch_geometric.transforms as T from torch_geometric.data import DataLoader from torch_geometric.nn import DynamicEdgeConv, global_max_pool from pointnet2_classification import MLP path = osp.join(osp.dirname(osp.realpath(__file__)), '..', 'data/ModelNet10') pre_transform, transform = T.NormalizeScale(), T.SamplePoints(1024) train_dataset = ModelNet(path, '10', True, transform, pre_transform) test_dataset = ModelNet(path, '10', False, transform, pre_transform) train_loader = DataLoader(train_dataset, batch_size=32, shuffle=True, num_workers=6) test_loader = DataLoader(test_dataset, batch_size=32, shuffle=False, num_workers=6) class Net(torch.nn.Module): def __init__(self, out_channels, k=20, aggr='max'): super().__init__() self.conv1 = DynamicEdgeConv(MLP([2 * 3, 64, 64, 64]), k, aggr) self.conv2 = DynamicEdgeConv(MLP([2 * 64, 128]), k, aggr) self.lin1 = MLP([128 + 64, 1024])
def val_loader(self): return DataLoader(self._val_data, shuffle=self._shuffle, batch_size=self._batch_size)
def train(): # get the parameters args = get_args() # decide the device device = torch.device( 'cuda:1' if torch.cuda.is_available() and args.cuda else 'cpu') # load dataset train_dataset = PPI(root='/home/amax/xsx/data/gnn_datas/PPI', split='train') val_dataset = PPI(root='/home/amax/xsx/data/gnn_datas/PPI', split='val') test_dataset = PPI(root='/home/amax/xsx/data/gnn_datas/PPI', split='test') train_loader = DataLoader(train_dataset, batch_size=1, shuffle=True) val_loader = DataLoader(val_dataset, batch_size=2, shuffle=False) test_loader = DataLoader(test_dataset, batch_size=2, shuffle=False) # data = dataset[0].to(device) # create the model and optimizer model = AGNN_PPI(train_dataset.num_features, args.hidden_dim, train_dataset.num_classes, args.dropout).to(device) optimizer = Adam(model.parameters(), lr=args.lr) criterion = torch.nn.BCEWithLogitsLoss() # the information which need to be recorded start_time = time.time() bad_counter = 0 best_valid_f1 = 0.0 best_epoch = 0 least_loss = float("inf") best_model = None # beging training for epoch in range(args.epochs): # the steps of training model.train() total_loss = 0.0 for data in train_loader: data.batch = None data = data.to(device) optimizer.zero_grad() output = model(data) loss = criterion(output, data.y) total_loss += loss.item() loss.backward() optimizer.step() avg_loss = total_loss / len(train_loader.dataset) f1 = validate(model, val_loader, device) # print('Epoch: {:04d}'.format(epoch + 1), 'f1: {:.4f}'.format(f1), 'loss: {:.4f}'.format(avg_loss)) if avg_loss < least_loss: least_loss = avg_loss best_epoch = epoch + 1 best_valid_f1 = f1 best_model = copy.deepcopy(model) bad_counter = 0 else: bad_counter += 1 if bad_counter >= args.patience: break print("Optimization Finished!") used_time = time.time() - start_time print("Total epochs: {:2d}".format(best_epoch + 100)) print("Best epochs: {:2d}".format(best_epoch)) # print("Time for each epoch: {:.2f}s".format(used_time / (best_epoch + args.patience))) print("Best epoch's validate f1: {:.2f}".format(best_valid_f1 * 100)) # test the best trained model test(best_model, test_loader, device) print("Total time elapsed: {:.2f}s".format(used_time))
def main(): # Training settings parser = argparse.ArgumentParser( description= 'PyTorch implementation of pre-training of graph neural networks') parser.add_argument('--device', type=int, default=0, help='which gpu to use if any (default: 0)') parser.add_argument('--batch_size', type=int, default=32, help='input batch size for training (default: 32)') parser.add_argument('--epochs', type=int, default=100, help='number of epochs to train (default: 100)') parser.add_argument('--lr', type=float, default=0.001, help='learning rate (default: 0.001)') parser.add_argument('--decay', type=float, default=0, help='weight decay (default: 0)') parser.add_argument( '--num_layer', type=int, default=5, help='number of GNN message passing layers (default: 5).') parser.add_argument('--emb_dim', type=int, default=300, help='embedding dimensions (default: 300)') parser.add_argument('--dropout_ratio', type=float, default=0.2, help='dropout ratio (default: 0.2)') parser.add_argument( '--graph_pooling', type=str, default="mean", help='graph level pooling (sum, mean, max, set2set, attention)') parser.add_argument( '--JK', type=str, default="last", help= 'how the node features across layers are combined. last, sum, max or concat' ) parser.add_argument( '--dataset', type=str, default='chembl_filtered', help='root directory of dataset. For now, only classification.') parser.add_argument('--gnn_type', type=str, default="gin") parser.add_argument('--input_model_file', type=str, default='', help='filename to read the model (if there is any)') parser.add_argument('--output_model_file', type=str, default='', help='filename to output the pre-trained model') parser.add_argument('--num_workers', type=int, default=8, help='number of workers for dataset loading') args = parser.parse_args() torch.manual_seed(0) np.random.seed(0) device = torch.device( "cuda:" + str(args.device)) if torch.cuda.is_available() else torch.device("cpu") if torch.cuda.is_available(): torch.cuda.manual_seed_all(0) #Bunch of classification tasks if args.dataset == "chembl_filtered": num_tasks = 1310 else: raise ValueError("Invalid dataset name.") #set up dataset dataset = MoleculeDataset("dataset/" + args.dataset, dataset=args.dataset) loader = DataLoader(dataset, batch_size=args.batch_size, shuffle=True, num_workers=args.num_workers) #set up model model = GNN_graphpred(args.num_layer, args.emb_dim, num_tasks, JK=args.JK, drop_ratio=args.dropout_ratio, graph_pooling=args.graph_pooling, gnn_type=args.gnn_type) if not args.input_model_file == "": model.from_pretrained(args.input_model_file + ".pth") model.to(device) #set up optimizer optimizer = optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.decay) print(optimizer) for epoch in range(1, args.epochs + 1): print("====epoch " + str(epoch)) train(args, model, device, loader, optimizer) if not args.output_model_file == "": torch.save(model.gnn.state_dict(), args.output_model_file + ".pth")
state = { 'epoch': epoch, 'state_dict': model_cpu, 'optimizer_state_dict': optimizer.state_dict(), 'scheduler_state_dict': scheduler.state_dict() } filename = '{}/{}_{}.pth'.format(save_path, name_pre, name_post) torch.save(state, filename) if __name__ == '__main__': opt = OptInit().initialize() opt.printer.info('===> Creating dataloader ...') test_dataset = GeoData.PPI(opt.data_dir, split='test') test_loader = DataLoader(test_dataset, batch_size=opt.batch_size, shuffle=True) opt.n_classes = test_loader.dataset.num_classes opt.printer.info('===> Loading the network ...') model = DeepGCN(opt).to(opt.device) if opt.multi_gpus: model = DataParallel(DeepGCN(opt)).to(opt.device) opt.printer.info('===> loading pre-trained ...') model, opt.best_value, opt.epoch = load_pretrained_models( model, opt.pretrained_model, opt.phase) if opt.phase == 'train': train_dataset = GeoData.PPI(opt.data_dir, 'train') if opt.multi_gpus: train_loader = DataListLoader(train_dataset,
print(dataset.num_classes) # print number of node features print(dataset.num_node_labels) # Access the graph in the dataset data = dataset[0] # There are 168/2 = 84 undirected edges # There are 37 nodes with 3 features # There is 1 class print(data) # split dataset to train_dataset and test_dataset train_dataset = dataset[:540] test_dataset = dataset[540:] # random permutation dataset = dataset.shuffle # set dataloader loader = DataLoader(dataset, batch_size=32, shuffle=True) for data in loader: print(data) print(data.num_graphs) print(data.num_node_features) # Averages data.x values with the same batch index x = scatter_mean(data.x, data.batch, dim=0)
showers = HGCalShowers(root='/beegfs/desy/user/korcariw/hgcal_model/', raw_files=['ntupleTree_0.root'], out_file='photonShowers50_100GeV_0.pt', include_labels=True, load_on_gpu=True) device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") print(f'Model will be loaded on {device}..') print('Splitting the data.. ') train_dataset = showers[:180000] test_dataset = showers[180000:190000] val_dataset = showers[190000:] train_loader = DataLoader(train_dataset, batch_size=hyperp['batch_size'], shuffle=True) test_loader = DataLoader(test_dataset, shuffle=False) val_loader = DataLoader(val_dataset, batch_size=hyperp['batch_size'], shuffle=False) print("Done.") class GCN(torch.nn.Module): def __init__(self, hidden_channels): super(GCN, self).__init__() torch.manual_seed(126755) self.bnorm = BatchNorm(4)