def main(): train_data = read_train() random.shuffle(train_data) test = train_data[1500:] train_data = train_data[:1500] ini = make_initial() gnn = GNN(15, 8, ini['x']) EPOCHS = 100 ini["param"]["m"] = {"W": np.zeros_like(ini["param"]["W"]), "A": np.zeros_like(ini["param"]["A"]), "b": np.zeros_like( ini["param"]["b"])} ini["param"]["v"] = copy.deepcopy(ini["param"]["m"]) ini["param"]["step"] = 0 adgd = adam(gnn, 50, train_data, 0.001, 0.9, 0.999, ini["param"], 2, EPOCHS, test) ''' with open('adgd.pickle', 'wb') as handle: pickle.dump(adgd, handle, protocol=pickle.HIGHEST_PROTOCOL) with open('msgd.pickle', 'rb') as handle: dmsgd = pickle.load(handle) with open('adgd.pickle', 'rb') as handle: adgd = pickle.load(handle) plot(adgd, dmsgd) ''' test = read_test() ans = [] for i in range(NUM_TEST): ans.append(gnn.predict(adgd["param"], 2, test[i]["adjacency_matrix"])) with open('prediction.txt', 'w') as f: for item in ans: f.write("%s\n" % item)
def approach_2(args): if args.model_name == './pretrained_models/sum_sum_binary.model': # change the default model for regression args.model_name = "./pretrained_models/sum_sum_regression.model" device = chainer.get_device(args.device) # load the trained model, model attributes and graph dataset with open('{}_stat'.format(args.model_name), mode='rb') as f: model_args = pickle.load(f) dataset = util.GraphData("mixed", model_args.feat_init_method, "Regression", args.data_num, device) # convert tuple dataset to graphs and targets graphs, targets = dataset.converter(dataset, device) model = GNN(model_args.num_layers, model_args.num_mlp_layers, dataset.graphs[0].node_features.shape[1], model_args.hidden_dim, dataset.graphs[0].node_features.shape[1], model_args.final_dropout, model_args.graph_pooling_type, model_args.neighbor_pooling_type, model_args.task_type) chainer.serializers.load_npz(args.model_name, model) model.to_device(device) device.use() print('\n--- Prediction by approach2 ---') # predict treewidth eval_st = time.time() with chainer.using_config('train', False), chainer.using_config( 'enable_backprop', False): raw_output = model(graphs) prediction = np.round(raw_output.array).reshape(len(raw_output)) eval_time = time.time() - eval_st # calculate some stats print('# of graphs\t{0}'.format(str(len(graphs)).rjust(5))) print('Execution time\t{0}'.format(eval_time)) print('Execution time per each graphs\t{0}'.format(eval_time / len(graphs))) print('Mean Absolute Error\t{0}'.format( sklearn.metrics.mean_absolute_error(targets, prediction))) print('Max Error\t{0}'.format( sklearn.metrics.max_error(targets, prediction))) # output the scatter plot sns.set_style("whitegrid", {'grid.linestyle': '--'}) df = pd.DataFrame({"Real Value": targets, "Predict Value": prediction}) g = sns.jointplot(df["Real Value"], df["Predict Value"]) g.ax_joint.plot([49, 1], [49, 1], ':k') # Please make this directory before run this code... plt.savefig('./{0}/Approach2/scatter.png'.format(args.out))
def main(): N = 4 # 頂点の数 D = 4 # dimension edge = [[1, 2], [2, 3], [2, 4], [3, 4]] # Edge x = np.array([[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]]) W = np.array([ [0.5, 0.3, 0, 0], [0.2, 0.1, 0, -0.1], [-0.4, -0.5, 1, 0], [-3, 0, 0, 1], ]) y = 1 g = GNN(N, D, x) alpha = 0.001 A = np.array([0.0, 0.0, 0.0, 0.0]).reshape(4, 1) b = 0 param = {"W": W, "A": A, "b": b} g.GD(alpha, param, y, 1, g.get_adjacency_matrix(edge))
def objective(trial): num_layers = trial.suggest_int('num_layers', 1, 10) num_mlp_layers = trial.suggest_int('num_mlp_layers', 1, 10) hidden_dim = trial.suggest_int('hidden_dim', 16, 128) final_dropout = trial.suggest_uniform('final_dropout', 0, 0.5) graph_pooling_type = trial.suggest_categorical('graph_pooling_type', ['max', 'average', 'sum']) neighbor_pooling_type = trial.suggest_categorical('neighbor_pooling_type', ['max', 'average', 'sum']) batchsize = trial.suggest_int('batchsize', 16, 128) device = chainer.get_device(-1) # Classification model = L.Classifier(GNN(num_layers, num_mlp_layers, dataset.graphs[0].node_features.shape[1], hidden_dim, dataset.graphs[0].node_features.shape[1], final_dropout, graph_pooling_type, neighbor_pooling_type, "Regression")) # choose the using device model.to_device(device) device.use() # Setup an optimizer optimizer = chainer.optimizers.Adam() optimizer.setup(model) # split the dataset into traindata and testdata train, test = chainer.datasets.split_dataset_random(dataset, int(dataset.__len__() * 0.9)) train_iter = chainer.iterators.SerialIterator(train, batchsize) test_iter = chainer.iterators.SerialIterator(test, batchsize, repeat=False, shuffle=False) # Set up a trainer updater = training.updaters.StandardUpdater(train_iter, optimizer, device=device, converter=dataset.converter) trainer = training.Trainer(updater, (300, 'epoch'), out= "result/hypara") # Evaluate the model with the test dataset for each epoch trainer.extend(extensions.Evaluator(test_iter, model, device=device, converter=dataset.converter)) trainer.extend(extensions.LogReport(filename='log_{}.dat'.format(trial.number))) trainer.extend(extensions.PlotReport(['main/loss', 'validation/main/loss'], 'epoch', file_name='loss_{}.png'.format(trial.number))) # Write a log of evaluation statistics for each epoch trainer.extend(extensions.PlotReport(['main/accuracy', 'validation/main/accuracy'], 'epoch', file_name='accuracy_{}.png'.format(trial.number))) # Run the training trainer.run() # save the model chainer.serializers.save_npz('./result/hypara/{0}.model'.format(trial.number), model) # return the AUC graphs, target = dataset.converter(test, device) with chainer.using_config('train', False), chainer.using_config('enable_backprop', False): y_pred = model.predictor(graphs) y_pred.to_cpu() y_pred = y_pred.array target = chainer.cuda.to_cpu(target) return 1 - sklearn.metrics.roc_auc_score(target, y_pred[:,1])
def __init__(self, num_entities, num_node_features, num_edge_features, config=None, entity_names=None, n_regions=2): self.n_regions = n_regions GNN.__init__(self, num_entities, num_node_features, num_edge_features, config, entity_names) self.num_entities = num_entities self.activation = 'relu' self.config = config self.top_k = config.top_k self.weight_file_name = self.create_weight_file_name() self.optimizer = self.create_optimizer(config.optimizer, config.lr) self.num_node_features = num_node_features self.node_input, self.edge_input, self.action_input, self.op_input, self.cost_input = \ self.create_inputs(num_entities, num_node_features, num_edge_features, dim_action=[1]) self.n_node_features = num_node_features
def test(test_dataloader, model=None): if model is None: obj = torch.load(args.save_path + '.model', map_location=lambda storage, loc: storage) train_args = obj['args'] model = GNN(word_vocab_size=WORD_VOCAB_SIZE, char_vocab_size=CHAR_VOCAB_SIZE, d_output=d_output, args=train_args) model.load_state_dict(obj['model']) model.cuda() print('Model loaded.') test_acc, test_prec, test_recall, test_f1 = evaluate(model, test_dataloader, output=True, args=args) print('######## prec : ', acc_to_str(test_prec)) print('######## recall : ', acc_to_str(test_recall)) print('######## f1 : ', acc_to_str(test_f1)) prec, recall, f1 = np.mean(list(test_prec.values())), np.mean( list(test_recall.values())), np.mean(list(test_f1.values())) print(prec, recall, f1) result_obj['test_prec'] = prec result_obj['test_recall'] = recall result_obj['test_f1'] = f1 result_obj['test_info'] = '\n'.join( [acc_to_str(test_prec), acc_to_str(test_recall), acc_to_str(test_f1)])
def main(): train_data = read_train() random.shuffle(train_data) test = train_data[1500:] train_data = train_data[:1500] ini = make_initial() gnn = GNN(15, 8, ini['x']) EPOCHS = 100 dmsgd = momentum_sgd(gnn, 50, train_data, 0.001, ini["param"], 2, EPOCHS, 0.9, test) dsgd = sgd(gnn, 50, train_data, 0.001, ini["param"], 2, EPOCHS, test) with open('sgd.pickle', 'wb') as handle: pickle.dump(dsgd, handle, protocol=pickle.HIGHEST_PROTOCOL) with open('msgd.pickle', 'wb') as handle: pickle.dump(dmsgd, handle, protocol=pickle.HIGHEST_PROTOCOL) with open('sgd.pickle', 'rb') as handle: dsgd = pickle.load(handle) with open('msgd.pickle', 'rb') as handle: dmsgd = pickle.load(handle) plot(dsgd, dmsgd)
def approach_1(args): device = chainer.get_device(args.device) # load the trained model, model attributes and graph dataset with open('{}_stat'.format(args.model_name), mode='rb') as f: model_args = pickle.load(f) dataset = util.GraphData("mixed", model_args.feat_init_method, "Regression", args.data_num, device) model = chainer.links.Classifier(GNN(model_args.num_layers, model_args.num_mlp_layers, dataset.graphs[0].node_features.shape[1], model_args.hidden_dim, 2, model_args.final_dropout, model_args.graph_pooling_type, model_args.neighbor_pooling_type, model_args.task_type)) chainer.serializers.load_npz(args.model_name, model) model.to_device(device) device.use() print('\n--- Prediction by existing algorithm ---') print('calctype\tIndex\t|V|\t|E|\ttw(G)\ttime\tevaltw\tfunccallnum') output_file = "exist.dat" result = ["ID\t|V|\t|E|\ttw\ttime\tevaltw\tfunccallnum"] for calc in ["upper", "lower"]: for idx in range(0, len(dataset.graphs)): eval_graph = dataset.graphs[idx] graphstat = "{4}\t{3}\t{0}\t{1}\t{2}".format(eval_graph.g.number_of_nodes(), eval_graph.g.number_of_edges(), dataset.labels[idx], str(idx).rjust(5), calc) print(graphstat, end='\t') try: tm, evtw, fcn = ordinaryDP(eval_graph, calc) res = "{0}\t{1}\t{2}".format(tm, evtw, fcn) except timeout_decorator.TimeoutError: res = "TimeOut" print(res) result.append(graphstat + "\t" + res) # write results to a file if args.out_to_file: with open("./{}/Approach1/".format(args.out) + output_file, "w") as f: f.write('\n'.join(result))
def readout(): N = 4 # 頂点の数 D = 4 # dimension edge = [[1, 2], [2, 3], [2, 4], [3, 4]] # Edge x = np.array([[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]]) W = np.array([ [0.5, 0.3, 0, 0], [0.2, 0.1, 0, -0.1], [-0.4, -0.5, 1, 0], [-3, 0, 0, 1], ]) g = GNN(N, D, x) a = g.aggregate_1(x, g.get_adjacency_matrix(edge)) # print(a) # print(W @ a) nx = g.aggregate_2(W, a) # print(nx) h = g.readout(nx) return h
def main(): # Training settings parser = argparse.ArgumentParser( description='GNN baselines on pcqm4m with DGL') parser.add_argument('--seed', type=int, default=42, help='random seed to use (default: 42)') parser.add_argument('--device', type=int, default=0, help='which gpu to use if any (default: 0)') parser.add_argument( '--gnn', type=str, default='gin-virtual', help='GNN to use, which can be from ' '[gin, gin-virtual, gcn, gcn-virtual] (default: gin-virtual)') parser.add_argument( '--graph_pooling', type=str, default='sum', help='graph pooling strategy mean or sum (default: sum)') parser.add_argument('--drop_ratio', type=float, default=0, help='dropout ratio (default: 0)') parser.add_argument( '--num_layers', type=int, default=5, help='number of GNN message passing layers (default: 5)') parser.add_argument( '--emb_dim', type=int, default=600, help='dimensionality of hidden units in GNNs (default: 600)') parser.add_argument('--batch_size', type=int, default=256, help='input batch size for training (default: 256)') parser.add_argument('--num_workers', type=int, default=0, help='number of workers (default: 0)') parser.add_argument('--checkpoint_dir', type=str, default='', help='directory to save checkpoint') parser.add_argument('--save_test_dir', type=str, default='', help='directory to save test submission file') args = parser.parse_args() print(args) np.random.seed(args.seed) torch.manual_seed(args.seed) random.seed(args.seed) if torch.cuda.is_available(): torch.cuda.manual_seed(args.seed) device = torch.device("cuda:" + str(args.device)) else: device = torch.device("cpu") ### automatic data loading and splitting ### Read in the raw SMILES strings smiles_dataset = PCQM4MDataset(root='dataset/', only_smiles=True) split_idx = smiles_dataset.get_idx_split() test_smiles_dataset = [smiles_dataset[i] for i in split_idx['test']] onthefly_dataset = OnTheFlyPCQMDataset(test_smiles_dataset) test_loader = DataLoader(onthefly_dataset, batch_size=args.batch_size, shuffle=False, num_workers=args.num_workers, collate_fn=collate_dgl) ### automatic evaluator. evaluator = PCQM4MEvaluator() shared_params = { 'num_layers': args.num_layers, 'emb_dim': args.emb_dim, 'drop_ratio': args.drop_ratio, 'graph_pooling': args.graph_pooling } if args.gnn == 'gin': model = GNN(gnn_type='gin', virtual_node=False, **shared_params).to(device) elif args.gnn == 'gin-virtual': model = GNN(gnn_type='gin', virtual_node=True, **shared_params).to(device) elif args.gnn == 'gcn': model = GNN(gnn_type='gcn', virtual_node=False, **shared_params).to(device) elif args.gnn == 'gcn-virtual': model = GNN(gnn_type='gcn', virtual_node=True, **shared_params).to(device) else: raise ValueError('Invalid GNN type') num_params = sum(p.numel() for p in model.parameters()) print(f'#Params: {num_params}') checkpoint_path = os.path.join(args.checkpoint_dir, 'checkpoint.pt') if not os.path.exists(checkpoint_path): raise RuntimeError(f'Checkpoint file not found at {checkpoint_path}') ## reading in checkpoint checkpoint = torch.load(checkpoint_path) model.load_state_dict(checkpoint['model_state_dict']) print('Predicting on test data...') y_pred = test(model, device, test_loader) print('Saving test submission file...') evaluator.save_test_submission({'y_pred': y_pred}, args.save_test_dir)
max_it = 50 num_epoch = 5000 optimizer = tf.train.AdamOptimizer # initialize state and output network net = n.Net(input_dim, state_dim, output_dim) # initialize GNN param = "st_d" + str(state_dim) + "_th" + str(threshold) + "_lr" + str( learning_rate) print(param) g = GNN.GNN(net, max_it=max_it, input_dim=input_dim, output_dim=output_dim, state_dim=state_dim, optimizer=optimizer, learning_rate=learning_rate, threshold=threshold, param=param, config=config) count = 0 # train the model and validate every 30 epochs for j in range(0, num_epoch): g.Train(inp[0], arcnode[0], labels, count, nodegraph[0]) if count % 30 == 0: print("Epoch ", count) print("Training: ", g.Validate(inp[0], arcnode[0], labels, count, nodegraph[0])) print(
def main(): device = torch.device("cuda:" + str(args.device)) if torch.cuda.is_available() else torch.device("cpu") ### automatic dataloading and splitting dataset = PygGraphPropPredDataset(name = args.dataset, root='/cmlscratch/kong/datasets/ogb') seq_len_list = np.array([len(seq) for seq in dataset.data.y]) print('Target seqence less or equal to {} is {}%.'.format(args.max_seq_len, np.sum(seq_len_list <= args.max_seq_len) / len(seq_len_list))) split_idx = dataset.get_idx_split() # print(split_idx['train']) # print(split_idx['valid']) # print(split_idx['test']) # train_method_name = [' '.join(dataset.data.y[i]) for i in split_idx['train']] # valid_method_name = [' '.join(dataset.data.y[i]) for i in split_idx['valid']] # test_method_name = [' '.join(dataset.data.y[i]) for i in split_idx['test']] # print('#train') # print(len(train_method_name)) # print('#valid') # print(len(valid_method_name)) # print('#test') # print(len(test_method_name)) # train_method_name_set = set(train_method_name) # valid_method_name_set = set(valid_method_name) # test_method_name_set = set(test_method_name) # # unique method name # print('#unique train') # print(len(train_method_name_set)) # print('#unique valid') # print(len(valid_method_name_set)) # print('#unique test') # print(len(test_method_name_set)) # # unique valid/test method name # print('#valid unseen during training') # print(len(valid_method_name_set - train_method_name_set)) # print('#test unseen during training') # print(len(test_method_name_set - train_method_name_set)) ### building vocabulary for sequence predition. Only use training data. vocab2idx, idx2vocab = get_vocab_mapping([dataset.data.y[i] for i in split_idx['train']], args.num_vocab) # test encoder and decoder # for data in dataset: # # PyG >= 1.5.0 # print(data.y) # # # PyG 1.4.3 # # print(data.y[0]) # data = encode_y_to_arr(data, vocab2idx, args.max_seq_len) # print(data.y_arr[0]) # decoded_seq = decode_arr_to_seq(data.y_arr[0], idx2vocab) # print(decoded_seq) # print('') ## test augment_edge # data = dataset[2] # print(data) # data_augmented = augment_edge(data) # print(data_augmented) ### set the transform function # augment_edge: add next-token edge as well as inverse edges. add edge attributes. # encode_y_to_arr: add y_arr to PyG data object, indicating the array representation of a sequence. dataset.transform = transforms.Compose([augment_edge, lambda data: encode_y_to_arr(data, vocab2idx, args.max_seq_len)]) ### automatic evaluator. takes dataset name as input evaluator = Evaluator(args.dataset) train_loader = DataLoader(dataset[split_idx["train"]], batch_size=args.batch_size, shuffle=True, num_workers = args.num_workers) valid_loader = DataLoader(dataset[split_idx["valid"]], batch_size=args.batch_size, shuffle=False, num_workers = args.num_workers) test_loader = DataLoader(dataset[split_idx["test"]], batch_size=args.batch_size, shuffle=False, num_workers = args.num_workers) nodetypes_mapping = pd.read_csv(os.path.join(dataset.root, 'mapping', 'typeidx2type.csv.gz')) nodeattributes_mapping = pd.read_csv(os.path.join(dataset.root, 'mapping', 'attridx2attr.csv.gz')) ### Encoding node features into emb_dim vectors. ### The following three node features are used. # 1. node type # 2. node attribute # 3. node depth node_encoder = ASTNodeEncoder(args.emb_dim, num_nodetypes = len(nodetypes_mapping['type']), num_nodeattributes = len(nodeattributes_mapping['attr']), max_depth = 20) vals, tests = [], [] for run in range(args.runs): best_val, final_test = 0, 0 if args.gnn == 'gin': model = GNN(num_vocab=len(vocab2idx), max_seq_len=args.max_seq_len, node_encoder=node_encoder, num_layer=args.num_layer, gnn_type='gin', emb_dim=args.emb_dim, drop_ratio=args.drop_ratio, virtual_node=False).to(device) elif args.gnn == 'gin-virtual': model = GNN(num_vocab=len(vocab2idx), max_seq_len=args.max_seq_len, node_encoder=node_encoder, num_layer=args.num_layer, gnn_type='gin', emb_dim=args.emb_dim, drop_ratio=args.drop_ratio, virtual_node=True).to(device) elif args.gnn == 'gcn': model = GNN(num_vocab=len(vocab2idx), max_seq_len=args.max_seq_len, node_encoder=node_encoder, num_layer=args.num_layer, gnn_type='gcn', emb_dim=args.emb_dim, drop_ratio=args.drop_ratio, virtual_node=False).to(device) elif args.gnn == 'gcn-virtual': model = GNN(num_vocab=len(vocab2idx), max_seq_len=args.max_seq_len, node_encoder=node_encoder, num_layer=args.num_layer, gnn_type='gcn', emb_dim=args.emb_dim, drop_ratio=args.drop_ratio, virtual_node=True).to(device) else: raise ValueError('Invalid GNN type') optimizer = torch.optim.Adam(model.parameters(), lr=args.lr) for epoch in range(1, args.epochs+1): loss = train(model, device, train_loader, optimizer, args) if epoch > args.epochs // 2 and epoch % args.test_freq == 0 or epoch == args.epochs: #4min train_perf = eval(model, device, train_loader, evaluator, arr_to_seq=lambda arr: decode_arr_to_seq(arr, idx2vocab)) valid_perf = eval(model, device, valid_loader, evaluator, arr_to_seq=lambda arr: decode_arr_to_seq(arr, idx2vocab)) test_perf = eval(model, device, test_loader, evaluator, arr_to_seq=lambda arr: decode_arr_to_seq(arr, idx2vocab)) result = (train_perf[dataset.eval_metric], valid_perf[dataset.eval_metric], test_perf[dataset.eval_metric]) _, val, tst = result if val > best_val: best_val = val final_test = tst print(f'Run{run} val:{best_val}, test:{final_test}') vals.append(best_val) tests.append(final_test) print('') print(f"Average val accuracy: {np.mean(vals)} ± {np.std(vals)}") print(f"Average test accuracy: {np.mean(tests)} ± {np.std(tests)}")
target_p = torch.zeros(opt['num_node'], opt['num_class']) if opt['cuda']: inputs = inputs.cuda() target = target.cuda() idx_train = idx_train.cuda() idx_dev = idx_dev.cuda() idx_test = idx_test.cuda() idx_all = idx_all.cuda() idx_unlabeled = idx_unlabeled.cuda() inputs_q = inputs_q.cuda() target_q = target_q.cuda() inputs_p = inputs_p.cuda() target_p = target_p.cuda() gnn = GNN(opt, adj) trainer = Trainer(opt, gnn) # Build the ema model gnn_ema = GNN(opt, adj) for ema_param, param in zip(gnn_ema.parameters(), gnn.parameters()): ema_param.data= param.data for param in gnn_ema.parameters(): param.detach_() trainer_ema = Trainer(opt, gnn_ema, ema = False) def init_data(): inputs_q.copy_(inputs)
def main(): # Training settings parser = argparse.ArgumentParser( description="GNN baselines on pcqm4m with Pytorch Geometrics") parser.add_argument("--device", type=int, default=0, help="which gpu to use if any (default: 0)") parser.add_argument( "--gnn", type=str, default="gin-virtual", help= "GNN gin, gin-virtual, or gcn, or gcn-virtual (default: gin-virtual)", ) parser.add_argument( "--graph_pooling", type=str, default="sum", help="graph pooling strategy mean or sum (default: sum)", ) parser.add_argument("--drop_ratio", type=float, default=0, help="dropout ratio (default: 0)") parser.add_argument( "--num_layers", type=int, default=5, help="number of GNN message passing layers (default: 5)", ) parser.add_argument( "--emb_dim", type=int, default=600, help="dimensionality of hidden units in GNNs (default: 600)", ) parser.add_argument("--train_subset", action="store_true") parser.add_argument( "--batch_size", type=int, default=256, help="input batch size for training (default: 256)", ) parser.add_argument( "--epochs", type=int, default=100, help="number of epochs to train (default: 100)", ) parser.add_argument("--num_workers", type=int, default=0, help="number of workers (default: 0)") parser.add_argument("--log_dir", type=str, default="", help="tensorboard log directory") parser.add_argument("--checkpoint_dir", type=str, default="", help="directory to save checkpoint") parser.add_argument( "--save_test_dir", type=str, default="", help="directory to save test submission file", ) args = parser.parse_args() print(args) np.random.seed(42) torch.manual_seed(42) torch.cuda.manual_seed(42) random.seed(42) device = (torch.device("cuda:" + str(args.device)) if torch.cuda.is_available() else torch.device("cpu")) ### automatic dataloading and splitting dataset = PygPCQM4MDataset(root="dataset/") split_idx = dataset.get_idx_split() ### automatic evaluator. takes dataset name as input evaluator = PCQM4MEvaluator() if args.train_subset: subset_ratio = 0.1 subset_idx = torch.randperm(len( split_idx["train"]))[:int(subset_ratio * len(split_idx["train"]))] train_loader = DataLoader( dataset[split_idx["train"][subset_idx]], batch_size=args.batch_size, shuffle=True, num_workers=args.num_workers, ) else: train_loader = DataLoader( dataset[split_idx["train"]], batch_size=args.batch_size, shuffle=True, num_workers=args.num_workers, ) valid_loader = DataLoader( dataset[split_idx["valid"]], batch_size=args.batch_size, shuffle=False, num_workers=args.num_workers, ) if args.save_test_dir is not "": test_loader = DataLoader( dataset[split_idx["test"]], batch_size=args.batch_size, shuffle=False, num_workers=args.num_workers, ) if args.checkpoint_dir is not "": os.makedirs(args.checkpoint_dir, exist_ok=True) shared_params = { "num_layers": args.num_layers, "emb_dim": args.emb_dim, "drop_ratio": args.drop_ratio, "graph_pooling": args.graph_pooling, } if args.gnn == "gin": model = GNN(gnn_type="gin", virtual_node=False, **shared_params).to(device) elif args.gnn == "gin-virtual": model = GNN(gnn_type="gin", virtual_node=True, **shared_params).to(device) elif args.gnn == "gcn": model = GNN(gnn_type="gcn", virtual_node=False, **shared_params).to(device) elif args.gnn == "gcn-virtual": model = GNN(gnn_type="gcn", virtual_node=True, **shared_params).to(device) else: raise ValueError("Invalid GNN type") num_params = sum(p.numel() for p in model.parameters()) print(f"#Params: {num_params}") optimizer = optim.Adam(model.parameters(), lr=0.001) if args.log_dir is not "": writer = SummaryWriter(log_dir=args.log_dir) best_valid_mae = 1000 if args.train_subset: scheduler = StepLR(optimizer, step_size=300, gamma=0.25) args.epochs = 1000 else: scheduler = StepLR(optimizer, step_size=30, gamma=0.25) for epoch in range(1, args.epochs + 1): print("=====Epoch {}".format(epoch)) print("Training...") train_mae = train(model, device, train_loader, optimizer) print("Evaluating...") valid_mae = eval(model, device, valid_loader, evaluator) print({"Train": train_mae, "Validation": valid_mae}) if args.log_dir is not "": writer.add_scalar("valid/mae", valid_mae, epoch) writer.add_scalar("train/mae", train_mae, epoch) if valid_mae < best_valid_mae: best_valid_mae = valid_mae if args.checkpoint_dir is not "": print("Saving checkpoint...") checkpoint = { "epoch": epoch, "model_state_dict": model.state_dict(), "optimizer_state_dict": optimizer.state_dict(), "scheduler_state_dict": scheduler.state_dict(), "best_val_mae": best_valid_mae, "num_params": num_params, } torch.save(checkpoint, os.path.join(args.checkpoint_dir, "checkpoint.pt")) if args.save_test_dir is not "": print("Predicting on test data...") y_pred = test(model, device, test_loader) print("Saving test submission file...") evaluator.save_test_submission({"y_pred": y_pred}, args.save_test_dir) scheduler.step() print(f"Best validation MAE so far: {best_valid_mae}") if args.log_dir is not "": writer.close()
input_dim = inp.shape[1] output_dim = labels.shape[1] max_it = 50 num_epoch = 10000 optimizer = tf.train.AdamOptimizer # initialize state and output network net = n.Net(input_dim, state_dim, output_dim) # initialize GNN param = "st_d" + str(state_dim) + "_th" + str(threshold) + "_lr" + str(learning_rate) print(param) tensorboard = False g = GNN.GNN(net, input_dim, output_dim, state_dim, max_it, optimizer, learning_rate, threshold, graph_based=False, param=param, config=config, tensorboard=tensorboard) # train the model count = 0 ###### for j in range(0, num_epoch): _, it = g.Train(inputs=inp, ArcNode=arcnode, target=labels, step=count) #if count % 30 == 0: if count % 3000 == 0: print("Epoch ", count) print("Training: ", g.Validate(inp, arcnode, labels, count)) # end = time.time()
def main(): seed = args.seed np.random.seed(seed) random.seed(seed) torch.manual_seed(seed) torch.cuda.manual_seed(seed) torch.cuda.manual_seed_all(seed) model_save_dir = f'models/{args.name}' os.makedirs(model_save_dir, exist_ok=True) device = torch.device( "cuda:" + str(args.device)) if torch.cuda.is_available() else torch.device("cpu") print("Training") # writer = SummaryWriter(model_save_dir) with open(f'{model_save_dir}/arguments.txt', 'w') as f: json.dump(args.__dict__, f, indent=2) ### automatic dataloading and splitting dataset = PygGraphPropPredDataset(name=args.dataset, transform=add_zeros) split_idx = dataset.get_idx_split() ### automatic evaluator. takes dataset name as input evaluator = Evaluator(args.dataset) train_loader = DataLoader(dataset[split_idx["train"]], batch_size=args.batch_size, shuffle=True, num_workers=args.num_workers) valid_loader = DataLoader(dataset[split_idx["valid"]], batch_size=args.batch_size, shuffle=False, num_workers=args.num_workers) test_loader = DataLoader(dataset[split_idx["test"]], batch_size=args.batch_size, shuffle=False, num_workers=args.num_workers) vals, tests = [], [] for run in range(args.runs): best_val, final_test = 0, 0 if args.gnn == 'gin': model = GNN(gnn_type='gin', num_class=dataset.num_classes, num_layer=args.num_layer, emb_dim=args.emb_dim, drop_ratio=args.drop_ratio, virtual_node=False, topological=args.topological).to(device) elif args.gnn == 'gin-virtual': model = GNN(gnn_type='gin', num_class=dataset.num_classes, num_layer=args.num_layer, emb_dim=args.emb_dim, drop_ratio=args.drop_ratio, virtual_node=True, topological=args.topological).to(device) elif args.gnn == 'gcn': model = GNN(gnn_type='gcn', num_class=dataset.num_classes, num_layer=args.num_layer, emb_dim=args.emb_dim, drop_ratio=args.drop_ratio, virtual_node=False, topological=args.topological).to(device) elif args.gnn == 'gcn-virtual': model = GNN(gnn_type='gcn', num_class=dataset.num_classes, num_layer=args.num_layer, emb_dim=args.emb_dim, drop_ratio=args.drop_ratio, virtual_node=True, topological=args.topological).to(device) elif args.gnn == 'controller': model = ControllerTransformer().to(device) else: raise ValueError('Invalid GNN type') optimizer = torch.optim.Adam(model.parameters(), lr=args.lr) for epoch in range(1, args.epochs + 1): loss = train(model, device, train_loader, optimizer, args) if epoch > args.epochs // 2 and epoch % args.test_freq == 0 or epoch == args.epochs: # 4min train_perf = eval(model, device, train_loader, evaluator) valid_perf = eval(model, device, valid_loader, evaluator) test_perf = eval(model, device, test_loader, evaluator) result = (train_perf[dataset.eval_metric], valid_perf[dataset.eval_metric], test_perf[dataset.eval_metric]) _, val, tst = result if val > best_val: torch.save(model.state_dict(), os.path.join(model_save_dir, f'model-best.pth')) best_val = val final_test = tst print(f'Run{run} val:{best_val}, test:{final_test}') vals.append(best_val) tests.append(final_test) print('') print(f"Average val accuracy: {np.mean(vals)} ± {np.std(vals)}") print(f"Average test accuracy: {np.mean(tests)} ± {np.std(tests)}")
def main(): ############# training set ########## set_name = "cli_15_7_200" ############# training set ################ #inp, arcnode, nodegraph, nodein, labels = Library.set_load_subgraph(data_path, "train") inp, arcnode, nodegraph, nodein, labels, _ = gnn_utils.set_load_general( data_path, "train", set_name=set_name) ############ test set #################### #inp_test, arcnode_test, nodegraph_test, nodein_test, labels_test = Library.set_load_subgraph(data_path, "test") inp_test, arcnode_test, nodegraph_test, nodein_test, labels_test, _ = gnn_utils.set_load_general( data_path, "test", set_name=set_name) ############ validation set ############# #inp_val, arcnode_val, nodegraph_val, nodein_val, labels_val = Library.set_load_subgraph(data_path, "valid") inp_val, arcnode_val, nodegraph_val, nodein_val, labels_val, _ = gnn_utils.set_load_general( data_path, "validation", set_name=set_name) # set threshold, learning rate and state dimension threshold = 0.001 learning_rate = 0.01 state_dim = 5 # set input and output dim, the maximum number of iterations, the number of epochs and the optimizer tf.reset_default_graph() input_dim = len(inp[0][0]) output_dim = 2 max_it = 50 num_epoch = 5000 optimizer = tf.train.AdamOptimizer # initialize state and output network net = n.Net(input_dim, state_dim, output_dim) # initialize GNN param = "st_d" + str(state_dim) + "_th" + str(threshold) + "_lr" + str( learning_rate) print(param) g = GNN.GNN(net, max_it=max_it, input_dim=input_dim, output_dim=output_dim, state_dim=state_dim, optimizer=optimizer, learning_rate=learning_rate, threshold=threshold, param=param, config=config) count = 0 # train the model and validate every 30 epochs for j in range(0, num_epoch): g.Train(inp[0], arcnode[0], labels, count, nodegraph[0]) if count % 30 == 0: print("Epoch ", count) print("Training: ", g.Validate(inp[0], arcnode[0], labels, count, nodegraph[0])) print( "Validation: ", g.Validate(inp_val[0], arcnode_val[0], labels_val, count, nodegraph_val[0])) count = count + 1 # evaluate on the test set print( g.Evaluate(inp_test[0], arcnode_test[0], labels_test, nodegraph_test[0]))
def main(): # Training settings parser = argparse.ArgumentParser( description='GNN baselines on pcqm4m with DGL') parser.add_argument('--seed', type=int, default=42, help='random seed to use (default: 42)') parser.add_argument('--device', type=int, default=0, help='which gpu to use if any (default: 0)') parser.add_argument( '--gnn', type=str, default='gin-virtual', help='GNN to use, which can be from ' '[gin, gin-virtual, gcn, gcn-virtual] (default: gin-virtual)') parser.add_argument( '--graph_pooling', type=str, default='sum', help='graph pooling strategy mean or sum (default: sum)') parser.add_argument('--drop_ratio', type=float, default=0, help='dropout ratio (default: 0)') parser.add_argument( '--num_layers', type=int, default=5, help='number of GNN message passing layers (default: 5)') parser.add_argument( '--emb_dim', type=int, default=600, help='dimensionality of hidden units in GNNs (default: 600)') parser.add_argument('--train_subset', action='store_true', help='use 10% of the training set for training') parser.add_argument('--batch_size', type=int, default=256, help='input batch size for training (default: 256)') parser.add_argument('--epochs', type=int, default=100, help='number of epochs to train (default: 100)') parser.add_argument('--num_workers', type=int, default=0, help='number of workers (default: 0)') parser.add_argument('--log_dir', type=str, default="", help='tensorboard log directory. If not specified, ' 'tensorboard will not be used.') parser.add_argument('--checkpoint_dir', type=str, default='', help='directory to save checkpoint') parser.add_argument('--save_test_dir', type=str, default='', help='directory to save test submission file') args = parser.parse_args() print(args) np.random.seed(args.seed) torch.manual_seed(args.seed) random.seed(args.seed) if torch.cuda.is_available(): torch.cuda.manual_seed(args.seed) device = torch.device("cuda:" + str(args.device)) else: device = torch.device("cpu") ### automatic dataloading and splitting dataset = SampleDglPCQM4MDataset(root='dataset/') # split_idx['train'], split_idx['valid'], split_idx['test'] # separately gives a 1D int64 tensor split_idx = dataset.get_idx_split() split_idx["train"] = split_idx["train"].type(torch.LongTensor) split_idx["test"] = split_idx["test"].type(torch.LongTensor) split_idx["valid"] = split_idx["valid"].type(torch.LongTensor) ### automatic evaluator. evaluator = PCQM4MEvaluator() if args.train_subset: subset_ratio = 0.1 subset_idx = torch.randperm(len( split_idx["train"]))[:int(subset_ratio * len(split_idx["train"]))] train_loader = DataLoader(dataset[split_idx["train"][subset_idx]], batch_size=args.batch_size, shuffle=True, num_workers=args.num_workers, collate_fn=collate_dgl) else: train_loader = DataLoader(dataset[split_idx["train"]], batch_size=args.batch_size, shuffle=True, num_workers=args.num_workers, collate_fn=collate_dgl) valid_loader = DataLoader(dataset[split_idx["valid"]], batch_size=args.batch_size, shuffle=False, num_workers=args.num_workers, collate_fn=collate_dgl) if args.save_test_dir is not '': test_loader = DataLoader(dataset[split_idx["test"]], batch_size=args.batch_size, shuffle=False, num_workers=args.num_workers, collate_fn=collate_dgl) if args.checkpoint_dir is not '': os.makedirs(args.checkpoint_dir, exist_ok=True) shared_params = { 'num_layers': args.num_layers, 'emb_dim': args.emb_dim, 'drop_ratio': args.drop_ratio, 'graph_pooling': args.graph_pooling } if args.gnn == 'gin': model = GNN(gnn_type='gin', virtual_node=False, **shared_params).to(device) elif args.gnn == 'gin-virtual': model = GNN(gnn_type='gin', virtual_node=True, **shared_params).to(device) elif args.gnn == 'gcn': model = GNN(gnn_type='gcn', virtual_node=False, **shared_params).to(device) elif args.gnn == 'gcn-virtual': model = GNN(gnn_type='gcn', virtual_node=True, **shared_params).to(device) elif args.gnn == 'gin-virtual-diffpool': model = DiffPoolGNN(gnn_type='gin', virtual_node=True, **shared_params).to(device) elif args.gnn == 'gin-virtual-bayes-diffpool': model = BayesDiffPoolGNN(gnn_type='gin', virtual_node=True, **shared_params).to(device) else: raise ValueError('Invalid GNN type') num_params = sum(p.numel() for p in model.parameters()) print(f'#Params: {num_params}') optimizer = optim.Adam(model.parameters(), lr=0.001) if args.log_dir is not '': writer = SummaryWriter(log_dir=args.log_dir) best_valid_mae = 1000 if args.train_subset: scheduler = StepLR(optimizer, step_size=300, gamma=0.25) args.epochs = 1000 else: scheduler = StepLR(optimizer, step_size=30, gamma=0.25) """ load from latest checkpoint """ # start epoch (default = 1, unless resuming training) firstEpoch = 1 # check if checkpoint exist -> load model checkpointFile = os.path.join(args.checkpoint_dir, 'checkpoint.pt') if os.path.exists(checkpointFile): # load checkpoint file checkpointData = torch.load(checkpointFile) firstEpoch = checkpointData["epoch"] model.load_state_dict(checkpointData["model_state_dict"]) optimizer.load_state_dict(checkpointData["optimizer_state_dict"]) scheduler.load_state_dict(checkpointData["scheduler_state_dict"]) best_valid_mae = checkpointData["best_val_mae"] num_params = checkpointData["num_params"] print( "Loaded existing weights from {}. Continuing from epoch: {} with best valid MAE: {}" .format(checkpointFile, firstEpoch, best_valid_mae)) for epoch in range(firstEpoch, args.epochs + 1): print("=====Epoch {}".format(epoch)) print('Training...') train_mae = train(model, device, train_loader, optimizer, args.gnn) print('Evaluating...') valid_mae = eval(model, device, valid_loader, evaluator) print({'Train': train_mae, 'Validation': valid_mae}) if args.log_dir is not '': writer.add_scalar('valid/mae', valid_mae, epoch) writer.add_scalar('train/mae', train_mae, epoch) if valid_mae < best_valid_mae: best_valid_mae = valid_mae if args.checkpoint_dir is not '': print('Saving checkpoint...') checkpoint = { 'epoch': epoch, 'model_state_dict': model.state_dict(), 'optimizer_state_dict': optimizer.state_dict(), 'scheduler_state_dict': scheduler.state_dict(), 'best_val_mae': best_valid_mae, 'num_params': num_params } torch.save(checkpoint, os.path.join(args.checkpoint_dir, 'checkpoint.pt')) if args.save_test_dir is not '': print('Predicting on test data...') y_pred = test(model, device, test_loader) print('Saving test submission file...') evaluator.save_test_submission({'y_pred': y_pred}, args.save_test_dir) scheduler.step() print(f'Best validation MAE so far: {best_valid_mae}') if args.log_dir is not '': writer.close()
''' PFN internship 2019 coding task machine learning task-1 Issei NAKASONE ''' import datasets as D from gnn import GNN filepath = '../datasets/train/0_graph.txt' graph = D.read_graph(filepath) model = GNN() h_G, _ = model.forward(graph, T=2) print(h_G)
''' PFN internship 2019 coding task machine learning task-3 Issei NAKASONE ''' import datasets as D import optimizers as op from gnn import GNN, TrainGNN from iterator import Iterator dirpath = '../datasets/train/' batch_size = 128 train, test = D.get_dataset(dirpath, test_ratio=0.25) train_iter = Iterator(train, batch_size) test_iter = Iterator(test, batch_size) model = GNN() optimizer = op.SGD() #optimizer = op.MomentumSGD() optimizer.setup(model) trainer = TrainGNN(optimizer, train_iter, test_iter) trainer.start(epoch=50)
def train(seed): print('random seed:', seed) torch.manual_seed(seed) torch.cuda.manual_seed(seed) # torch.backends.cudnn.enabled = False dataset = read_data('../data', '../graph') label2id = dataset.label2id print(label2id) vocab_size = dataset.vocab_size output_dim = len(label2id) def acc_to_str(acc): s = ['%s:%.3f' % (label, acc[label]) for label in acc] return '{' + ', '.join(s) + '}' cross_res = {label: [] for label in label2id if label != 'O'} output_file = open('%s.mistakes' % args.output, 'w') for cross_valid in range(5): model = GNN(vocab_size=vocab_size, output_dim=output_dim, args=args) model.cuda() # print vocab_size dataset.split_train_valid_test([0.8, 0.1, 0.1], 5, cross_valid) print('train:', len(dataset.train), 'valid:', len(dataset.valid), 'test:', len(dataset.test)) def evaluate(model, datalist, output_file=None): if output_file != None: output_file.write( '#############################################\n') correct = {label: 0 for label in label2id if label != 'O'} total = len(datalist) model.eval() print_cnt = 0 for data in datalist: word, feat = Variable(data.input_word).cuda(), Variable( data.input_feat).cuda() a_ud, a_lr = Variable(data.a_ud, requires_grad=False).cuda(), Variable( data.a_lr, requires_grad=False).cuda() mask = Variable(data.mask, requires_grad=False).cuda() if args.globalnode: logprob, form = model(word, feat, mask, a_ud, a_lr) logprob = logprob.data.view(-1, output_dim) else: logprob = model(word, feat, mask, a_ud, a_lr).data.view(-1, output_dim) mask = mask.data.view(-1) y_pred = torch.LongTensor(output_dim) for i in range(output_dim): prob = logprob[:, i].exp() * mask y_pred[i] = prob.topk(k=1)[1][0] # y_pred = logprob.topk(k=1,dim=0)[1].view(-1) for label in label2id: if label == 'O': continue labelid = label2id[label] if data.output.view(-1)[y_pred[labelid]] == labelid: correct[label] += 1 else: if output_file != None: num_sent, sent_len, word_len = data.input_word.size( ) id = y_pred[label2id[label]] word = data.words[data.sents[int( id / sent_len)][id % sent_len]] output_file.write( '%d %d %s %s\n' % (data.set_id, data.fax_id, label, word)) return {label: float(correct[label]) / total for label in correct} batch = 1 weight = torch.zeros(len(label2id)) for label, id in label2id.items(): weight[id] = 1 if label == 'O' else 10 loss_function = nn.NLLLoss(weight.cuda(), reduce=False) optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=args.lr / float(batch), weight_decay=args.wd) # scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=1, gamma=0.8) best_acc = -1 wait = 0 for epoch in range(args.epochs): sum_loss = 0 model.train() # random.shuffle(dataset.train) for idx, data in enumerate(dataset.train): word, feat = Variable(data.input_word).cuda(), Variable( data.input_feat).cuda() a_ud, a_lr = Variable(data.a_ud, requires_grad=False).cuda(), Variable( data.a_lr, requires_grad=False).cuda() mask = Variable(data.mask, requires_grad=False).cuda() true_output = Variable(data.output).cuda() if args.globalnode: logprob, form = model(word, feat, mask, a_ud, a_lr) else: logprob = model(word, feat, mask, a_ud, a_lr) loss = torch.mean( mask.view(-1) * loss_function(logprob.view(-1, output_dim), true_output.view(-1))) if args.globalnode: true_form = Variable(torch.LongTensor([data.set_id - 1 ])).cuda() loss = loss + 0.1 * F.nll_loss(form, true_form) sum_loss += loss.data.sum() loss.backward() if (idx + 1) % batch == 0 or idx + 1 == len(dataset.train): optimizer.step() optimizer.zero_grad() train_acc = evaluate(model, dataset.train) valid_acc = evaluate(model, dataset.valid) test_acc = evaluate(model, dataset.test) print('Epoch %d: Train Loss: %.3f Train: %s Valid: %s Test: %s' \ % (epoch, sum_loss, acc_to_str(train_acc), acc_to_str(valid_acc), acc_to_str(test_acc))) # scheduler.step() acc = np.log(list(valid_acc.values())).sum() if epoch < 6: continue if acc >= best_acc: torch.save(model.state_dict(), args.output + '.model') wait = 0 if acc > best_acc else wait + 1 best_acc = max(acc, best_acc) if wait >= args.patience: break model.load_state_dict(torch.load(args.output + '.model')) test_acc = evaluate(model, dataset.test, output_file=output_file) print('########', acc_to_str(test_acc)) for label in test_acc: cross_res[label].append(test_acc[label]) print("Cross Validation Result:") for label in cross_res: cross_res[label] = np.mean(cross_res[label]) print(acc_to_str(cross_res)) return cross_res
def main(): # Training settings parser = argparse.ArgumentParser( description='GNN baselines on pcqm4m with PGL') parser.add_argument('--use_cuda', action='store_true') parser.add_argument('--device', type=int, default=0, help='which gpu to use if any (default: 0)') parser.add_argument( '--gnn', type=str, default='gin-virtual', help= 'GNN gin, gin-virtual, or gcn, or gcn-virtual (default: gin-virtual)') parser.add_argument( '--graph_pooling', type=str, default='sum', help='graph pooling strategy mean or sum (default: sum)') parser.add_argument('--drop_ratio', type=float, default=0, help='dropout ratio (default: 0)') parser.add_argument( '--num_layers', type=int, default=5, help='number of GNN message passing layers (default: 5)') parser.add_argument( '--emb_dim', type=int, default=600, help='dimensionality of hidden units in GNNs (default: 600)') parser.add_argument('--train_subset', action='store_true') parser.add_argument('--batch_size', type=int, default=256, help='input batch size for training (default: 256)') parser.add_argument('--epochs', type=int, default=100, help='number of epochs to train (default: 100)') parser.add_argument('--num_workers', type=int, default=1, help='number of workers (default: 1)') parser.add_argument('--log_dir', type=str, default="", help='tensorboard log directory') parser.add_argument('--checkpoint_dir', type=str, default='', help='directory to save checkpoint') parser.add_argument('--save_test_dir', type=str, default='', help='directory to save test submission file') args = parser.parse_args() print(args) random.seed(42) np.random.seed(42) paddle.seed(42) if not args.use_cuda: paddle.set_device("cpu") ### automatic dataloading and splitting class Config(): def __init__(self): self.base_data_path = "./dataset" config = Config() ds = MolDataset(config) split_idx = ds.get_idx_split() test_ds = Subset(ds, split_idx['test']) print("Test exapmles: ", len(test_ds)) ### automatic evaluator. takes dataset name as input evaluator = PCQM4MEvaluator() test_loader = Dataloader(test_ds, batch_size=args.batch_size, shuffle=False, num_workers=args.num_workers, collate_fn=CollateFn()) shared_params = { 'num_layers': args.num_layers, 'emb_dim': args.emb_dim, 'drop_ratio': args.drop_ratio, 'graph_pooling': args.graph_pooling } if args.gnn == 'gin': model = GNN(gnn_type='gin', virtual_node=False, **shared_params) elif args.gnn == 'gin-virtual': model = GNN(gnn_type='gin', virtual_node=True, **shared_params) elif args.gnn == 'gcn': model = GNN(gnn_type='gcn', virtual_node=False, **shared_params) elif args.gnn == 'gcn-virtual': model = GNN(gnn_type='gcn', virtual_node=True, **shared_params) else: raise ValueError('Invalid GNN type') num_params = sum(p.numel() for p in model.parameters()) print(f'#Params: {num_params}') checkpoint_path = os.path.join(args.checkpoint_dir, 'checkpoint.pdparams') if not os.path.exists(checkpoint_path): raise RuntimeError(f'Checkpoint file not found at {checkpoint_path}') model.set_state_dict(paddle.load(checkpoint_path)) print('Predicting on test data...') y_pred = test(model, test_loader) print('Saving test submission file...') evaluator.save_test_submission({'y_pred': y_pred}, args.save_test_dir)
def main(): # Training settings parser = argparse.ArgumentParser(description='GNN baselines on ogbgmol* data with Pytorch Geometrics') parser.add_argument('--device', type=int, default=0, help='which gpu to use if any (default: 0)') parser.add_argument('--gnn', type=str, default='gin-virtual', help='GNN gin, gin-virtual, or gcn, or gcn-virtual (default: gin-virtual)') parser.add_argument('--drop_ratio', type=float, default=0.5, help='dropout ratio (default: 0.5)') parser.add_argument('--num_layer', type=int, default=5, help='number of GNN message passing layers (default: 5)') parser.add_argument('--pooling', type=str, default='mean', help='Pooling tecnhnique for graph embedding') parser.add_argument('--laf', type=str, default='mean', help='Init function if laf pooling is specified') parser.add_argument('--laf_layers', type=str, default='false', help='If set to true, internal layers will be initialized with laf function') parser.add_argument('--emb_dim', type=int, default=300, help='dimensionality of hidden units in GNNs (default: 300)') parser.add_argument('--batch_size', type=int, default=32, help='input batch size for training (default: 32)') parser.add_argument('--epochs', type=int, default=100, help='number of epochs to train (default: 100)') parser.add_argument('--num_workers', type=int, default=0, help='number of workers (default: 0)') parser.add_argument('--dataset', type=str, default="ogbg-molhiv", help='dataset name (default: ogbg-molhiv)') parser.add_argument('--feature', type=str, default="full", help='full feature or simple feature') parser.add_argument('--filename', type=str, default="", help='filename to output result (default: )') parser.add_argument('--seed', type=int, default=92, help='torch seed') parser.add_argument('--alternate', type=str, default='false', help='use alternate learning with laf') args = parser.parse_args() print(args) torch.manual_seed(args.seed) device = torch.device("cuda:" + str(args.device)) if torch.cuda.is_available() else torch.device("cpu") ### automatic dataloading and splitting dataset = PygGraphPropPredDataset(name=args.dataset) if args.feature == 'full': pass elif args.feature == 'simple': print('using simple feature') # only retain the top two node/edge features dataset.data.x = dataset.data.x[:, :2] dataset.data.edge_attr = dataset.data.edge_attr[:, :2] split_idx = dataset.get_idx_split() ### automatic evaluator. takes dataset name as input evaluator = Evaluator(args.dataset) train_loader = DataLoader(dataset[split_idx["train"]], batch_size=args.batch_size, shuffle=True, num_workers=args.num_workers) valid_loader = DataLoader(dataset[split_idx["valid"]], batch_size=args.batch_size, shuffle=False, num_workers=args.num_workers) test_loader = DataLoader(dataset[split_idx["test"]], batch_size=args.batch_size, shuffle=False, num_workers=args.num_workers) if args.gnn == 'gin': model = GNN(gnn_type='gin', num_tasks=dataset.num_tasks, emb_dim=args.emb_dim, drop_ratio=args.drop_ratio, virtual_node=False, graph_pooling=args.pooling, laf_fun=args.laf, laf_layers=args.laf_layers, device=device, lafgrad=True).to(device) elif args.gnn == 'gin-virtual': model = GNN(gnn_type='gin', num_tasks=dataset.num_tasks, emb_dim=args.emb_dim, drop_ratio=args.drop_ratio, virtual_node=True, graph_pooling=args.pooling, laf_fun=args.laf, laf_layers=args.laf_layers, device=device, lafgrad=True).to(device) elif args.gnn == 'gcn': model = GNN(gnn_type='gcn', num_tasks=dataset.num_tasks, emb_dim=args.emb_dim, drop_ratio=args.drop_ratio, virtual_node=False, graph_pooling=args.pooling, laf_fun=args.laf, laf_layers=args.laf_layers, device=device, lafgrad=True).to(device) elif args.gnn == 'gcn-virtual': model = GNN(gnn_type='gcn', num_tasks=dataset.num_tasks, emb_dim=args.emb_dim, drop_ratio=args.drop_ratio, virtual_node=True, graph_pooling=args.pooling, laf_fun=args.laf, laf_layers=args.laf_layers, device=device, lafgrad=True).to(device) elif args.gnn == 'gat': model = GNN(gnn_type='gat', num_tasks=dataset.num_tasks, emb_dim=args.emb_dim, drop_ratio=args.drop_ratio, virtual_node=False, graph_pooling=args.pooling, laf_fun=args.laf, laf_layers=args.laf_layers, device=device, lafgrad=True).to(device) else: raise ValueError('Invalid GNN type') #model.load_state_dict(torch.load("{}_fixed_training.mdl".format(args.filename))) model_params = [] laf_params = [] for n, p in model.named_parameters(): if n == 'pool.weights' or n == 'pool.alpha' or n == 'pool.beta' or n == 'pool.N' or n == 'pool.M': laf_params.append(p) else: model_params.append(p) optimizer = optim.Adam(model_params, lr=0.001) if laf_params == []: optimizerlaf = None else: optimizerlaf = optim.Adam(laf_params, lr=0.0001) flog = open(args.filename + ".log", 'a') valid_curve = [] test_curve = [] train_curve = [] if 'classification' in dataset.task_type: best_val = 0 else: best_val = 1e12 flog.write("{}\n".format(args)) bflag = True for epoch in range(1, args.epochs + 1): start = time.time() print("=====Epoch {}".format(epoch)) flog.write("=====Epoch {}\n".format(epoch)) print('Training...') #if args.alternate == 'false': # train_perf = train(model, device, train_loader, optimizer, optimizerlaf, dataset.task_type, evaluator) #else: # train_perf = train(model, device, train_loader, optimizer, None, dataset.task_type, evaluator) #if args.alternate == 'false': train_perf = train(model, device, train_loader, optimizer, optimizerlaf, dataset.task_type, evaluator, alternate=args.alternate) print('Evaluating...') # train_perf = eval(model, device, train_loader, evaluator) valid_perf = eval(model, device, valid_loader, evaluator) test_perf = eval(model, device, test_loader, evaluator) print({'Train': train_perf, 'Validation': valid_perf, 'Test': test_perf}) print("Time {:.4f}s".format(time.time() - start)) if laf_params != []: print("{}\n".format(torch.norm(model.pool.weights))) flog.write("{}\n".format({'Train': train_perf, 'Validation': valid_perf, 'Test': test_perf})) flog.write("Time: {}\n".format(time.time()-start)) if laf_params != []: flog.write("Laf weights norm: {}\n".format(torch.norm(model.pool.weights, dim=0))) flog.flush() train_curve.append(train_perf[dataset.eval_metric]) valid_curve.append(valid_perf[dataset.eval_metric]) test_curve.append(test_perf[dataset.eval_metric]) if 'classification' in dataset.task_type: if valid_perf[dataset.eval_metric] >= best_val: best_val = valid_perf[dataset.eval_metric] if not args.filename == '': if args.alternate == 'true': torch.save(model.state_dict(), '{}_fixed_training.mdl'.format(args.filename)) else: torch.save(model.state_dict(), '{}.mdl'.format(args.filename)) else: if valid_perf[dataset.eval_metric] <= best_val: best_val = epoch if not args.filename == '': if args.alternate == 'true': torch.save(model.state_dict(), '{}_fixed_training.mdl'.format(args.filename)) else: torch.save(model.state_dict(), '{}.mdl'.format(args.filename)) if 'classification' in dataset.task_type: best_val_epoch = np.argmax(np.array(valid_curve)) best_train = max(train_curve) else: best_val_epoch = np.argmin(np.array(valid_curve)) best_train = min(train_curve) print('Finished training!') print('Best validation score: {}'.format(valid_curve[best_val_epoch])) print('Test score: {}'.format(test_curve[best_val_epoch])) flog.write('Finished training!\n') flog.write('Best validation score: {}\n'.format(valid_curve[best_val_epoch])) flog.write('Test score: {}\n'.format(test_curve[best_val_epoch])) flog.flush() if not args.filename == '': torch.save({'Val': valid_curve[best_val_epoch], 'Test': test_curve[best_val_epoch], 'Train': train_curve[best_val_epoch], 'BestTrain': best_train}, args.filename + "_fixed_training.res") # if args.alternate == 'true'and optimizerlaf: # args.alternate = 'false' # flog.write("===================LAF TRAINING=================\n") # valid_curve = [] # test_curve = [] # train_curve = [] # if 'classification' in dataset.task_type: # best_val = 0 # else: # best_val = 1e12 # for epoch in range(1, args.epochs + 1): # start = time.time() # print("=====Epoch {}".format(epoch)) # flog.write("=====Epoch {}\n".format(epoch)) # print('Training...') # train_perf = train(model, device, train_loader, optimizerlaf, None, dataset.task_type, evaluator) # print('Evaluating...') # # train_perf = eval(model, device, train_loader, evaluator) # valid_perf = eval(model, device, valid_loader, evaluator) # test_perf = eval(model, device, test_loader, evaluator) # print({'Train': train_perf, 'Validation': valid_perf, 'Test': test_perf}) # print("Time {:.4f}s".format(time.time() - start)) # #print("{}\n".format(torch.norm(model.pool.weights))) # flog.write("{}\n".format({'Train': train_perf, 'Validation': valid_perf, 'Test': test_perf})) # flog.write("Time: {}\n".format(time.time()-start)) # #flog.write("Laf weights norm: {}\n".format(torch.norm(model.pool.weights, dim=0))) # flog.flush() # train_curve.append(train_perf[dataset.eval_metric]) # valid_curve.append(valid_perf[dataset.eval_metric]) # test_curve.append(test_perf[dataset.eval_metric]) # if 'classification' in dataset.task_type: # if valid_perf[dataset.eval_metric] >= best_val: # best_val = valid_perf[dataset.eval_metric] # if not args.filename == '': # torch.save(model.state_dict(), '{}_laf_training.mdl'.format(args.filename)) # else: # if valid_perf[dataset.eval_metric] <= best_val: # best_val = epoch # if not args.filename == '': # torch.save(model.state_dict(), '{}_laf_training.mdl'.format(args.filename)) # if 'classification' in dataset.task_type: # best_val_epoch = np.argmax(np.array(valid_curve)) # best_train = max(train_curve) # else: # best_val_epoch = np.argmin(np.array(valid_curve)) # best_train = min(train_curve) # print('Finished training!') # print('Best validation score: {}'.format(valid_curve[best_val_epoch])) # print('Test score: {}'.format(test_curve[best_val_epoch])) # flog.write('Finished training!\n') # flog.write('Best validation score: {}\n'.format(valid_curve[best_val_epoch])) # flog.write('Test score: {}\n'.format(test_curve[best_val_epoch])) # flog.flush() # if not args.filename == '': # torch.save({'Val': valid_curve[best_val_epoch], 'Test': test_curve[best_val_epoch], # 'Train': train_curve[best_val_epoch], 'BestTrain': best_train}, args.filename + "_laf_training.res") flog.close()
def main(): # Training settings parser = argparse.ArgumentParser( description='GNN baselines on pcqm4m with Pytorch Geometrics') parser.add_argument('--device', type=int, default=0, help='which gpu to use if any (default: 0)') parser.add_argument( '--gnn', type=str, default='gin-virtual', help= 'GNN gin, gin-virtual, or gcn, or gcn-virtual (default: gin-virtual)') parser.add_argument( '--graph_pooling', type=str, default='sum', help='graph pooling strategy mean or sum (default: sum)') parser.add_argument('--drop_ratio', type=float, default=0, help='dropout ratio (default: 0)') parser.add_argument( '--num_layers', type=int, default=5, help='number of GNN message passing layers (default: 5)') parser.add_argument( '--emb_dim', type=int, default=600, help='dimensionality of hidden units in GNNs (default: 600)') parser.add_argument('--train_subset', action='store_true') parser.add_argument('--batch_size', type=int, default=256, help='input batch size for training (default: 256)') parser.add_argument('--epochs', type=int, default=100, help='number of epochs to train (default: 100)') parser.add_argument('--num_workers', type=int, default=0, help='number of workers (default: 0)') parser.add_argument('--log_dir', type=str, default="", help='tensorboard log directory') parser.add_argument('--checkpoint_dir', type=str, default='', help='directory to save checkpoint') parser.add_argument('--save_test_dir', type=str, default='', help='directory to save test submission file') args = parser.parse_args() print(args) np.random.seed(42) torch.manual_seed(42) torch.cuda.manual_seed(42) random.seed(42) device = torch.device( "cuda:" + str(args.device)) if torch.cuda.is_available() else torch.device("cpu") ### automatic dataloading and splitting dataset = PygPCQM4MDataset(root='dataset/') split_idx = dataset.get_idx_split() ### automatic evaluator. takes dataset name as input evaluator = PCQM4MEvaluator() if args.train_subset: subset_ratio = 0.1 subset_idx = torch.randperm(len( split_idx["train"]))[:int(subset_ratio * len(split_idx["train"]))] train_loader = DataLoader(dataset[split_idx["train"][subset_idx]], batch_size=args.batch_size, shuffle=True, num_workers=args.num_workers) else: train_loader = DataLoader(dataset[split_idx["train"]], batch_size=args.batch_size, shuffle=True, num_workers=args.num_workers) valid_loader = DataLoader(dataset[split_idx["valid"]], batch_size=args.batch_size, shuffle=False, num_workers=args.num_workers) if args.save_test_dir != '': test_loader = DataLoader(dataset[split_idx["test-dev"]], batch_size=args.batch_size, shuffle=False, num_workers=args.num_workers) if args.checkpoint_dir != '': os.makedirs(args.checkpoint_dir, exist_ok=True) shared_params = { 'num_layers': args.num_layers, 'emb_dim': args.emb_dim, 'drop_ratio': args.drop_ratio, 'graph_pooling': args.graph_pooling } if args.gnn == 'gin': model = GNN(gnn_type='gin', virtual_node=False, **shared_params).to(device) elif args.gnn == 'gin-virtual': model = GNN(gnn_type='gin', virtual_node=True, **shared_params).to(device) elif args.gnn == 'gcn': model = GNN(gnn_type='gcn', virtual_node=False, **shared_params).to(device) elif args.gnn == 'gcn-virtual': model = GNN(gnn_type='gcn', virtual_node=True, **shared_params).to(device) else: raise ValueError('Invalid GNN type') num_params = sum(p.numel() for p in model.parameters()) print(f'#Params: {num_params}') optimizer = optim.Adam(model.parameters(), lr=0.001) if args.log_dir != '': writer = SummaryWriter(log_dir=args.log_dir) best_valid_mae = 1000 if args.train_subset: scheduler = StepLR(optimizer, step_size=300, gamma=0.25) args.epochs = 1000 else: scheduler = StepLR(optimizer, step_size=30, gamma=0.25) for epoch in range(1, args.epochs + 1): print("=====Epoch {}".format(epoch)) print('Training...') train_mae = train(model, device, train_loader, optimizer) print('Evaluating...') valid_mae = eval(model, device, valid_loader, evaluator) print({'Train': train_mae, 'Validation': valid_mae}) if args.log_dir != '': writer.add_scalar('valid/mae', valid_mae, epoch) writer.add_scalar('train/mae', train_mae, epoch) if valid_mae < best_valid_mae: best_valid_mae = valid_mae if args.checkpoint_dir != '': print('Saving checkpoint...') checkpoint = { 'epoch': epoch, 'model_state_dict': model.state_dict(), 'optimizer_state_dict': optimizer.state_dict(), 'scheduler_state_dict': scheduler.state_dict(), 'best_val_mae': best_valid_mae, 'num_params': num_params } torch.save(checkpoint, os.path.join(args.checkpoint_dir, 'checkpoint.pt')) if args.save_test_dir != '': print('Predicting on test data...') y_pred = test(model, device, test_loader) print('Saving test submission file...') evaluator.save_test_submission({'y_pred': y_pred}, args.save_test_dir, mode='test-dev') scheduler.step() print(f'Best validation MAE so far: {best_valid_mae}') if args.log_dir != '': writer.close()
def main(): device = torch.device( "cuda:" + str(args.device)) if torch.cuda.is_available() else torch.device("cpu") ### automatic dataloading and splitting dataset = PygGraphPropPredDataset(name=args.dataset, transform=add_zeros) split_idx = dataset.get_idx_split() ### automatic evaluator. takes dataset name as input evaluator = Evaluator(args.dataset) train_loader = DataLoader(dataset[split_idx["train"]], batch_size=args.batch_size, shuffle=True, num_workers=args.num_workers) valid_loader = DataLoader(dataset[split_idx["valid"]], batch_size=args.batch_size, shuffle=False, num_workers=args.num_workers) test_loader = DataLoader(dataset[split_idx["test"]], batch_size=args.batch_size, shuffle=False, num_workers=args.num_workers) vals, tests = [], [] for run in range(args.runs): best_val, final_test = 0, 0 if args.gnn == 'gin': model = GNN(gnn_type='gin', num_class=dataset.num_classes, num_layer=args.num_layer, emb_dim=args.emb_dim, drop_ratio=args.drop_ratio, virtual_node=False).to(device) elif args.gnn == 'gin-virtual': model = GNN(gnn_type='gin', num_class=dataset.num_classes, num_layer=args.num_layer, emb_dim=args.emb_dim, drop_ratio=args.drop_ratio, virtual_node=True).to(device) elif args.gnn == 'gcn': model = GNN(gnn_type='gcn', num_class=dataset.num_classes, num_layer=args.num_layer, emb_dim=args.emb_dim, drop_ratio=args.drop_ratio, virtual_node=False).to(device) elif args.gnn == 'gcn-virtual': model = GNN(gnn_type='gcn', num_class=dataset.num_classes, num_layer=args.num_layer, emb_dim=args.emb_dim, drop_ratio=args.drop_ratio, virtual_node=True).to(device) else: raise ValueError('Invalid GNN type') optimizer = torch.optim.Adam(model.parameters(), lr=args.lr) for epoch in range(1, args.epochs + 1): loss = train(model, device, train_loader, optimizer, args) if epoch > args.epochs // 2 and epoch % args.test_freq == 0 or epoch == args.epochs: #4min train_perf = eval(model, device, train_loader, evaluator) valid_perf = eval(model, device, valid_loader, evaluator) test_perf = eval(model, device, test_loader, evaluator) result = (train_perf[dataset.eval_metric], valid_perf[dataset.eval_metric], test_perf[dataset.eval_metric]) _, val, tst = result if val > best_val: best_val = val final_test = tst print(f'Run{run} val:{best_val}, test:{final_test}') vals.append(best_val) tests.append(final_test) print('') print(f"Average val accuracy: {np.mean(vals)} ± {np.std(vals)}") print(f"Average test accuracy: {np.mean(tests)} ± {np.std(tests)}")
def main(): # Training settings parser = argparse.ArgumentParser( description='GNN baselines on ogbg-code2 data with Pytorch Geometrics') parser.add_argument('--device', type=int, default=0, help='which gpu to use if any (default: 0)') parser.add_argument( '--gnn', type=str, default='gcn-virtual', help= 'GNN gin, gin-virtual, or gcn, or gcn-virtual (default: gcn-virtual)') parser.add_argument('--drop_ratio', type=float, default=0, help='dropout ratio (default: 0)') parser.add_argument('--max_seq_len', type=int, default=5, help='maximum sequence length to predict (default: 5)') parser.add_argument( '--num_vocab', type=int, default=5000, help= 'the number of vocabulary used for sequence prediction (default: 5000)' ) parser.add_argument( '--num_layer', type=int, default=5, help='number of GNN message passing layers (default: 5)') parser.add_argument( '--emb_dim', type=int, default=300, help='dimensionality of hidden units in GNNs (default: 300)') parser.add_argument('--batch_size', type=int, default=128, help='input batch size for training (default: 128)') parser.add_argument('--epochs', type=int, default=25, help='number of epochs to train (default: 25)') parser.add_argument('--random_split', action='store_true') parser.add_argument('--num_workers', type=int, default=0, help='number of workers (default: 0)') parser.add_argument('--dataset', type=str, default="ogbg-code2", help='dataset name (default: ogbg-code2)') parser.add_argument('--filename', type=str, default="", help='filename to output result (default: )') args = parser.parse_args() print(args) device = torch.device( "cuda:" + str(args.device)) if torch.cuda.is_available() else torch.device("cpu") ### automatic dataloading and splitting dataset = PygGraphPropPredDataset(name=args.dataset) seq_len_list = np.array([len(seq) for seq in dataset.data.y]) print('Target seqence less or equal to {} is {}%.'.format( args.max_seq_len, np.sum(seq_len_list <= args.max_seq_len) / len(seq_len_list))) split_idx = dataset.get_idx_split() if args.random_split: print('Using random split') perm = torch.randperm(len(dataset)) num_train, num_valid, num_test = len(split_idx['train']), len( split_idx['valid']), len(split_idx['test']) split_idx['train'] = perm[:num_train] split_idx['valid'] = perm[num_train:num_train + num_valid] split_idx['test'] = perm[num_train + num_valid:] assert (len(split_idx['train']) == num_train) assert (len(split_idx['valid']) == num_valid) assert (len(split_idx['test']) == num_test) # print(split_idx['train']) # print(split_idx['valid']) # print(split_idx['test']) # train_method_name = [' '.join(dataset.data.y[i]) for i in split_idx['train']] # valid_method_name = [' '.join(dataset.data.y[i]) for i in split_idx['valid']] # test_method_name = [' '.join(dataset.data.y[i]) for i in split_idx['test']] # print('#train') # print(len(train_method_name)) # print('#valid') # print(len(valid_method_name)) # print('#test') # print(len(test_method_name)) # train_method_name_set = set(train_method_name) # valid_method_name_set = set(valid_method_name) # test_method_name_set = set(test_method_name) # # unique method name # print('#unique train') # print(len(train_method_name_set)) # print('#unique valid') # print(len(valid_method_name_set)) # print('#unique test') # print(len(test_method_name_set)) # # unique valid/test method name # print('#valid unseen during training') # print(len(valid_method_name_set - train_method_name_set)) # print('#test unseen during training') # print(len(test_method_name_set - train_method_name_set)) ### building vocabulary for sequence predition. Only use training data. vocab2idx, idx2vocab = get_vocab_mapping( [dataset.data.y[i] for i in split_idx['train']], args.num_vocab) # test encoder and decoder # for data in dataset: # # PyG >= 1.5.0 # print(data.y) # # # PyG 1.4.3 # # print(data.y[0]) # data = encode_y_to_arr(data, vocab2idx, args.max_seq_len) # print(data.y_arr[0]) # decoded_seq = decode_arr_to_seq(data.y_arr[0], idx2vocab) # print(decoded_seq) # print('') ## test augment_edge # data = dataset[2] # print(data) # data_augmented = augment_edge(data) # print(data_augmented) ### set the transform function # augment_edge: add next-token edge as well as inverse edges. add edge attributes. # encode_y_to_arr: add y_arr to PyG data object, indicating the array representation of a sequence. dataset.transform = transforms.Compose([ augment_edge, lambda data: encode_y_to_arr(data, vocab2idx, args.max_seq_len) ]) ### automatic evaluator. takes dataset name as input evaluator = Evaluator(args.dataset) train_loader = DataLoader(dataset[split_idx["train"]], batch_size=args.batch_size, shuffle=True, num_workers=args.num_workers) valid_loader = DataLoader(dataset[split_idx["valid"]], batch_size=args.batch_size, shuffle=False, num_workers=args.num_workers) test_loader = DataLoader(dataset[split_idx["test"]], batch_size=args.batch_size, shuffle=False, num_workers=args.num_workers) nodetypes_mapping = pd.read_csv( os.path.join(dataset.root, 'mapping', 'typeidx2type.csv.gz')) nodeattributes_mapping = pd.read_csv( os.path.join(dataset.root, 'mapping', 'attridx2attr.csv.gz')) print(nodeattributes_mapping) ### Encoding node features into emb_dim vectors. ### The following three node features are used. # 1. node type # 2. node attribute # 3. node depth node_encoder = ASTNodeEncoder(args.emb_dim, num_nodetypes=len(nodetypes_mapping['type']), num_nodeattributes=len( nodeattributes_mapping['attr']), max_depth=20) if args.gnn == 'gin': model = GNN(num_vocab=len(vocab2idx), max_seq_len=args.max_seq_len, node_encoder=node_encoder, num_layer=args.num_layer, gnn_type='gin', emb_dim=args.emb_dim, drop_ratio=args.drop_ratio, virtual_node=False).to(device) elif args.gnn == 'gin-virtual': model = GNN(num_vocab=len(vocab2idx), max_seq_len=args.max_seq_len, node_encoder=node_encoder, num_layer=args.num_layer, gnn_type='gin', emb_dim=args.emb_dim, drop_ratio=args.drop_ratio, virtual_node=True).to(device) elif args.gnn == 'gcn': model = GNN(num_vocab=len(vocab2idx), max_seq_len=args.max_seq_len, node_encoder=node_encoder, num_layer=args.num_layer, gnn_type='gcn', emb_dim=args.emb_dim, drop_ratio=args.drop_ratio, virtual_node=False).to(device) elif args.gnn == 'gcn-virtual': model = GNN(num_vocab=len(vocab2idx), max_seq_len=args.max_seq_len, node_encoder=node_encoder, num_layer=args.num_layer, gnn_type='gcn', emb_dim=args.emb_dim, drop_ratio=args.drop_ratio, virtual_node=True).to(device) else: raise ValueError('Invalid GNN type') optimizer = optim.Adam(model.parameters(), lr=0.001) print(f'#Params: {sum(p.numel() for p in model.parameters())}') valid_curve = [] test_curve = [] train_curve = [] for epoch in range(1, args.epochs + 1): print("=====Epoch {}".format(epoch)) print('Training...') train(model, device, train_loader, optimizer) print('Evaluating...') train_perf = eval( model, device, train_loader, evaluator, arr_to_seq=lambda arr: decode_arr_to_seq(arr, idx2vocab)) valid_perf = eval( model, device, valid_loader, evaluator, arr_to_seq=lambda arr: decode_arr_to_seq(arr, idx2vocab)) test_perf = eval( model, device, test_loader, evaluator, arr_to_seq=lambda arr: decode_arr_to_seq(arr, idx2vocab)) print({ 'Train': train_perf, 'Validation': valid_perf, 'Test': test_perf }) train_curve.append(train_perf[dataset.eval_metric]) valid_curve.append(valid_perf[dataset.eval_metric]) test_curve.append(test_perf[dataset.eval_metric]) print('F1') best_val_epoch = np.argmax(np.array(valid_curve)) best_train = max(train_curve) print('Finished training!') print('Best validation score: {}'.format(valid_curve[best_val_epoch])) print('Test score: {}'.format(test_curve[best_val_epoch])) if not args.filename == '': result_dict = { 'Val': valid_curve[best_val_epoch], 'Test': test_curve[best_val_epoch], 'Train': train_curve[best_val_epoch], 'BestTrain': best_train } torch.save(result_dict, args.filename)
def main(opt): device = torch.device('cuda' if opt['cuda'] == True and torch.cuda.is_available() else 'cpu') #-------------------------------------------------- # Load data. #-------------------------------------------------- net_file = opt['dataset'] + '/net.txt' label_file = opt['dataset'] + '/label.txt' feature_file = opt['dataset'] + '/feature.txt' train_file = opt['dataset'] + '/train.txt' dev_file = opt['dataset'] + '/dev.txt' test_file = opt['dataset'] + '/test.txt' vocab_node = loader.Vocab(net_file, [0, 1]) vocab_label = loader.Vocab(label_file, [1]) vocab_feature = loader.Vocab(feature_file, [1]) opt['num_node'] = len(vocab_node) opt['num_feature'] = len(vocab_feature) opt['num_class'] = len(vocab_label) graph = loader.Graph(file_name=net_file, entity=[vocab_node, 0, 1]) label = loader.EntityLabel(file_name=label_file, entity=[vocab_node, 0], label=[vocab_label, 1]) feature = loader.EntityFeature(file_name=feature_file, entity=[vocab_node, 0], feature=[vocab_feature, 1]) d = graph.to_symmetric(opt['self_link_weight']) feature.to_one_hot(binary=True) adj = graph.get_sparse_adjacency(opt['cuda']) deg = torch.zeros(adj.shape[0]) for k,v in d.items(): deg[k] = v with open(train_file, 'r') as fi: idx_train = [vocab_node.stoi[line.strip()] for line in fi] with open(dev_file, 'r') as fi: idx_dev = [vocab_node.stoi[line.strip()] for line in fi] with open(test_file, 'r') as fi: idx_test = [vocab_node.stoi[line.strip()] for line in fi] inputs = torch.Tensor(feature.one_hot) target = torch.LongTensor(label.itol) idx_train = torch.LongTensor(idx_train) idx_dev = torch.LongTensor(idx_dev) idx_test = torch.LongTensor(idx_test) if opt['cuda']: inputs = inputs.cuda() target = target.cuda() idx_train = idx_train.cuda() idx_dev = idx_dev.cuda() idx_test = idx_test.cuda() #-------------------------------------------------- # Build model. #-------------------------------------------------- if opt['weight']: gnn = WGNN(opt, adj, deg, opt['time']) else: gnn = GNN(opt, adj, deg, opt['time']) trainer = Trainer(opt, gnn) print(gnn) print(opt) #-------------------------------------------------- # Train model. #-------------------------------------------------- def train(epochs): best = 0.0 results = [] prev_dev_acc = 0 cnt = 0 lr = opt['lr'] for epoch in range(0, epochs): # ----------------------- # Train Model # ----------------------- if opt['weight']: loss = trainer.updatew(inputs, target, idx_train) else: loss = trainer.update(inputs, target, idx_train) # ----------------------- # Evaluate Model # ----------------------- _, preds, accuracy_dev = trainer.evaluate(inputs, target, idx_dev) # ----------------------- # Test Model # ----------------------- _, preds, accuracy_test = trainer.evaluate(inputs, target, idx_test) print( 'Epoch: {} | Loss: {:.3f} | Dev acc: {:.3f} | Test acc: {:.3f} | Forward: {} {:.3f} | Backward: {} {:.3f}'.format( epoch, loss, accuracy_dev, accuracy_test, trainer.fm.get_value(), trainer.fm.get_average(), trainer.bm.get_value(), trainer.bm.get_average())) results += [(accuracy_dev, accuracy_test)] if accuracy_dev >= best: best = accuracy_dev state = dict([('model', copy.deepcopy(trainer.model.state_dict())), ('optim', copy.deepcopy(trainer.optimizer.state_dict()))]) trainer.model.load_state_dict(state['model']) trainer.optimizer.load_state_dict(state['optim']) return results results = train(opt['epoch']) def get_accuracy(results): best_dev, acc_test = 0.0, 0.0 for d, t in results: if d > best_dev: best_dev, acc_test = d, t return acc_test, best_dev acc_test = get_accuracy(results) print('{:.3f}'.format(acc_test[0]*100)) return acc_test
# retrieve input, arcnode, nodegraph and target for validation set inp_val = val[fold][0] arcnode_val = val[fold][1] labels_val = val[fold][4] nodegraph_val = val[fold][2] # initialize network net = n.Net(input_dim, state_dim, output_dim) # instantiate GNN g = GNN.GNN(net, input_dim, output_dim, state_dim, max_it, optimizer, learning_rate, threshold, graph_based=True, param=param, config=config) # train GNN, and validate every 2 epochs, (early stopping) count = 0 valid_best = None patience = 0 for j in range(0, num_epoch): g.Train(inp[0], arcnode[0], labels, count, nodegraph[0]) print("Epoch ", j) if count % 2 == 0:
def main(): # Training settings parser = argparse.ArgumentParser( description='GNN baselines on ogbg-ppi data with Pytorch Geometrics') parser.add_argument('--device', type=int, default=0, help='which gpu to use if any (default: 0)') parser.add_argument( '--gnn', type=str, default='gin-virtual', help= 'GNN gin, gin-virtual, or gcn, or gcn-virtual (default: gin-virtual)') parser.add_argument('--drop_ratio', type=float, default=0.5, help='dropout ratio (default: 0.5)') parser.add_argument( '--num_layer', type=int, default=5, help='number of GNN message passing layers (default: 5)') parser.add_argument( '--emb_dim', type=int, default=300, help='dimensionality of hidden units in GNNs (default: 300)') parser.add_argument('--batch_size', type=int, default=32, help='input batch size for training (default: 32)') parser.add_argument('--epochs', type=int, default=100, help='number of epochs to train (default: 100)') parser.add_argument('--num_workers', type=int, default=0, help='number of workers (default: 0)') parser.add_argument('--dataset', type=str, default="ogbg-ppi", help='dataset name (default: ogbg-ppi)') parser.add_argument('--filename', type=str, default="", help='filename to output result (default: )') args = parser.parse_args() device = torch.device( "cuda:" + str(args.device)) if torch.cuda.is_available() else torch.device("cpu") ### automatic dataloading and splitting dataset = PygGraphPropPredDataset(name=args.dataset, transform=add_zeros) splitted_idx = dataset.get_idx_split() ### automatic evaluator. takes dataset name as input evaluator = Evaluator(args.dataset) train_loader = DataLoader(dataset[splitted_idx["train"]], batch_size=args.batch_size, shuffle=True, num_workers=args.num_workers) valid_loader = DataLoader(dataset[splitted_idx["valid"]], batch_size=args.batch_size, shuffle=False, num_workers=args.num_workers) test_loader = DataLoader(dataset[splitted_idx["test"]], batch_size=args.batch_size, shuffle=False, num_workers=args.num_workers) if args.gnn == 'gin': model = GNN(gnn_type='gin', num_class=37, emb_dim=args.emb_dim, drop_ratio=args.drop_ratio, virtual_node=False).to(device) elif args.gnn == 'gin-virtual': model = GNN(gnn_type='gin', num_class=37, emb_dim=args.emb_dim, drop_ratio=args.drop_ratio, virtual_node=True).to(device) elif args.gnn == 'gcn': model = GNN(gnn_type='gcn', num_class=37, emb_dim=args.emb_dim, drop_ratio=args.drop_ratio, virtual_node=False).to(device) elif args.gnn == 'gcn-virtual': model = GNN(gnn_type='gcn', num_class=37, emb_dim=args.emb_dim, drop_ratio=args.drop_ratio, virtual_node=True).to(device) else: raise ValueError('Invalid GNN type') optimizer = optim.Adam(model.parameters(), lr=0.001) valid_curve = [] test_curve = [] train_curve = [] for epoch in range(1, args.epochs + 1): print("=====Epoch {}".format(epoch)) print('Training...') train(model, device, train_loader, optimizer) print('Evaluating...') train_perf = eval(model, device, train_loader, evaluator) valid_perf = eval(model, device, valid_loader, evaluator) test_perf = eval(model, device, test_loader, evaluator) print({ 'Train': train_perf, 'Validation': valid_perf, 'Test': test_perf }) train_curve.append(train_perf['acc']) valid_curve.append(valid_perf['acc']) test_curve.append(test_perf['acc']) best_val_epoch = np.argmax(np.array(valid_curve)) best_train = max(train_curve) print('Finished training!') print('Best validation score: {}'.format(valid_curve[best_val_epoch])) print('Test score: {}'.format(test_curve[best_val_epoch])) if not args.filename == '': torch.save( { 'Val': valid_curve[best_val_epoch], 'Test': test_curve[best_val_epoch], 'Train': train_curve[best_val_epoch], 'BestTrain': best_train }, args.filename)
def main(): # Training settings parser = argparse.ArgumentParser( description='GNN baselines on ogbg-ppa data with Pytorch Geometrics') parser.add_argument('--device', type=int, default=0, help='which gpu to use if any (default: 0)') parser.add_argument( '--gnn', type=str, default='gin-virtual', help= 'GNN gin, gin-virtual, or gcn, or gcn-virtual (default: gin-virtual)') parser.add_argument('--drop_ratio', type=float, default=0.5, help='dropout ratio (default: 0.5)') parser.add_argument( '--num_layer', type=int, default=5, help='number of GNN message passing layers (default: 5)') parser.add_argument('--pooling', type=str, default='mean', help='Pooling tecnhnique for graph embedding') parser.add_argument('--laf', type=str, default='mean', help='Init function if laf pooling is specified') parser.add_argument( '--laf_layers', type=str, default='false', help= 'If set to true, internal layers will be initialized with laf function' ) parser.add_argument( '--emb_dim', type=int, default=300, help='dimensionality of hidden units in GNNs (default: 300)') parser.add_argument('--batch_size', type=int, default=32, help='input batch size for training (default: 32)') parser.add_argument('--epochs', type=int, default=100, help='number of epochs to train (default: 100)') parser.add_argument('--num_workers', type=int, default=0, help='number of workers (default: 0)') parser.add_argument('--dataset', type=str, default="ogbg-ppa", help='dataset name (default: ogbg-ppa)') parser.add_argument('--filename', type=str, default="", help='filename to output result (default: )') parser.add_argument('--seed', type=int, default=92, help='torch seed') args = parser.parse_args() print(args) device = torch.device( "cuda:" + str(args.device)) if torch.cuda.is_available() else torch.device("cpu") ### automatic dataloading and splitting dataset = PygGraphPropPredDataset(name=args.dataset, transform=add_zeros) split_idx = dataset.get_idx_split() ### automatic evaluator. takes dataset name as input evaluator = Evaluator(args.dataset) train_loader = DataLoader(dataset[split_idx["train"]], batch_size=args.batch_size, shuffle=True, num_workers=args.num_workers) valid_loader = DataLoader(dataset[split_idx["valid"]], batch_size=args.batch_size, shuffle=False, num_workers=args.num_workers) test_loader = DataLoader(dataset[split_idx["test"]], batch_size=args.batch_size, shuffle=False, num_workers=args.num_workers) if args.gnn == 'gin': model = GNN(gnn_type='gin', emb_dim=args.emb_dim, drop_ratio=args.drop_ratio, virtual_node=False, graph_pooling=args.pooling, laf_fun=args.laf, device=args.device).to(device) elif args.gnn == 'gin-virtual': model = GNN(gnn_type='gin', emb_dim=args.emb_dim, drop_ratio=args.drop_ratio, virtual_node=True, graph_pooling=args.pooling, laf_fun=args.laf, device=args.device).to(device) elif args.gnn == 'gcn': model = GNN(gnn_type='gcn', emb_dim=args.emb_dim, drop_ratio=args.drop_ratio, virtual_node=False, graph_pooling=args.pooling, laf_fun=args.laf, device=args.device).to(device) elif args.gnn == 'gcn-virtual': model = GNN(gnn_type='gcn', emb_dim=args.emb_dim, drop_ratio=args.drop_ratio, virtual_node=True, graph_pooling=args.pooling, laf_fun=args.laf, device=args.device).to(device) else: raise ValueError('Invalid GNN type') optimizer = optim.Adam(model.parameters(), lr=0.001) valid_curve = [] test_curve = [] train_curve = [] best_val = 0 flog = open(args.filename + ".log", 'w') flog.write("{}\n".format(args)) for epoch in range(1, args.epochs + 1): start = time.time() print("=====Epoch {}".format(epoch)) flog.write("=====Epoch {}\n".format(epoch)) print('Training...') train(model, device, train_loader, optimizer) print('Evaluating...') train_perf = eval(model, device, train_loader, evaluator) valid_perf = eval(model, device, valid_loader, evaluator) test_perf = eval(model, device, test_loader, evaluator) print({ 'Train': train_perf, 'Validation': valid_perf, 'Test': test_perf }) print("Time {:.4f}s".format(time.time() - start)) flog.write("{}\tTime: {}s\n".format( { 'Train': train_perf, 'Validation': valid_perf, 'Test': test_perf }, time.time() - start)) flog.flush() train_curve.append(train_perf['acc']) valid_curve.append(valid_perf['acc']) test_curve.append(test_perf['acc']) if valid_perf[dataset.eval_metric] >= best_val: best_val = valid_perf[dataset.eval_metric] if not args.filename == '': torch.save(model.state_dict(), '{}.mdl'.format(args.filename)) best_val_epoch = np.argmax(np.array(valid_curve)) best_train = max(train_curve) print('Finished training!') print('Best validation score: {}'.format(valid_curve[best_val_epoch])) print('Test score: {}'.format(test_curve[best_val_epoch])) flog.write('Finished training!\n') flog.write('Best validation score: {}\n'.format( valid_curve[best_val_epoch])) flog.write('Test score: {}\n'.format(test_curve[best_val_epoch])) flog.flush() if not args.filename == '': torch.save( { 'Val': valid_curve[best_val_epoch], 'Test': test_curve[best_val_epoch], 'Train': train_curve[best_val_epoch], 'BestTrain': best_train }, args.filename + ".res")