def main():
train_data = read_train()
random.shuffle(train_data)
test = train_data[1500:]
train_data = train_data[:1500]
ini = make_initial()
gnn = GNN(15, 8, ini['x'])
EPOCHS = 100
ini["param"]["m"] = {"W": np.zeros_like(ini["param"]["W"]), "A": np.zeros_like(ini["param"]["A"]),
"b": np.zeros_like(
ini["param"]["b"])}
ini["param"]["v"] = copy.deepcopy(ini["param"]["m"])
ini["param"]["step"] = 0
adgd = adam(gnn, 50, train_data, 0.001, 0.9, 0.999,
ini["param"], 2, EPOCHS, test)
'''
with open('adgd.pickle', 'wb') as handle:
pickle.dump(adgd, handle, protocol=pickle.HIGHEST_PROTOCOL)
with open('msgd.pickle', 'rb') as handle:
dmsgd = pickle.load(handle)
with open('adgd.pickle', 'rb') as handle:
adgd = pickle.load(handle)
plot(adgd, dmsgd)
'''
test = read_test()
ans = []
for i in range(NUM_TEST):
ans.append(gnn.predict(adgd["param"], 2, test[i]["adjacency_matrix"]))
with open('prediction.txt', 'w') as f:
for item in ans:
f.write("%s\n" % item)
def approach_2(args):
if args.model_name == './pretrained_models/sum_sum_binary.model':
# change the default model for regression
args.model_name = "./pretrained_models/sum_sum_regression.model"
device = chainer.get_device(args.device)
# load the trained model, model attributes and graph dataset
with open('{}_stat'.format(args.model_name), mode='rb') as f:
model_args = pickle.load(f)
dataset = util.GraphData("mixed", model_args.feat_init_method,
"Regression", args.data_num, device)
# convert tuple dataset to graphs and targets
graphs, targets = dataset.converter(dataset, device)
model = GNN(model_args.num_layers, model_args.num_mlp_layers,
dataset.graphs[0].node_features.shape[1],
model_args.hidden_dim,
dataset.graphs[0].node_features.shape[1],
model_args.final_dropout, model_args.graph_pooling_type,
model_args.neighbor_pooling_type, model_args.task_type)
chainer.serializers.load_npz(args.model_name, model)
model.to_device(device)
device.use()
print('\n--- Prediction by approach2 ---')
# predict treewidth
eval_st = time.time()
with chainer.using_config('train', False), chainer.using_config(
'enable_backprop', False):
raw_output = model(graphs)
prediction = np.round(raw_output.array).reshape(len(raw_output))
eval_time = time.time() - eval_st
# calculate some stats
print('# of graphs\t{0}'.format(str(len(graphs)).rjust(5)))
print('Execution time\t{0}'.format(eval_time))
print('Execution time per each graphs\t{0}'.format(eval_time /
len(graphs)))
print('Mean Absolute Error\t{0}'.format(
sklearn.metrics.mean_absolute_error(targets, prediction)))
print('Max Error\t{0}'.format(
sklearn.metrics.max_error(targets, prediction)))
# output the scatter plot
sns.set_style("whitegrid", {'grid.linestyle': '--'})
df = pd.DataFrame({"Real Value": targets, "Predict Value": prediction})
g = sns.jointplot(df["Real Value"], df["Predict Value"])
g.ax_joint.plot([49, 1], [49, 1], ':k')
# Please make this directory before run this code...
plt.savefig('./{0}/Approach2/scatter.png'.format(args.out))
def main():
N = 4 # 頂点の数
D = 4 # dimension
edge = [[1, 2], [2, 3], [2, 4], [3, 4]] # Edge
x = np.array([[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]])
W = np.array([
[0.5, 0.3, 0, 0],
[0.2, 0.1, 0, -0.1],
[-0.4, -0.5, 1, 0],
[-3, 0, 0, 1],
])
y = 1
g = GNN(N, D, x)
alpha = 0.001
A = np.array([0.0, 0.0, 0.0, 0.0]).reshape(4, 1)
b = 0
param = {"W": W, "A": A, "b": b}
g.GD(alpha, param, y, 1, g.get_adjacency_matrix(edge))
def objective(trial):
num_layers = trial.suggest_int('num_layers', 1, 10)
num_mlp_layers = trial.suggest_int('num_mlp_layers', 1, 10)
hidden_dim = trial.suggest_int('hidden_dim', 16, 128)
final_dropout = trial.suggest_uniform('final_dropout', 0, 0.5)
graph_pooling_type = trial.suggest_categorical('graph_pooling_type', ['max', 'average', 'sum'])
neighbor_pooling_type = trial.suggest_categorical('neighbor_pooling_type', ['max', 'average', 'sum'])
batchsize = trial.suggest_int('batchsize', 16, 128)
device = chainer.get_device(-1)
# Classification
model = L.Classifier(GNN(num_layers, num_mlp_layers, dataset.graphs[0].node_features.shape[1],
hidden_dim, dataset.graphs[0].node_features.shape[1], final_dropout, graph_pooling_type, neighbor_pooling_type, "Regression"))
# choose the using device
model.to_device(device)
device.use()
# Setup an optimizer
optimizer = chainer.optimizers.Adam()
optimizer.setup(model)
# split the dataset into traindata and testdata
train, test = chainer.datasets.split_dataset_random(dataset, int(dataset.__len__() * 0.9))
train_iter = chainer.iterators.SerialIterator(train, batchsize)
test_iter = chainer.iterators.SerialIterator(test, batchsize, repeat=False, shuffle=False)
# Set up a trainer
updater = training.updaters.StandardUpdater(train_iter, optimizer, device=device, converter=dataset.converter)
trainer = training.Trainer(updater, (300, 'epoch'), out= "result/hypara")
# Evaluate the model with the test dataset for each epoch
trainer.extend(extensions.Evaluator(test_iter, model, device=device, converter=dataset.converter))
trainer.extend(extensions.LogReport(filename='log_{}.dat'.format(trial.number)))
trainer.extend(extensions.PlotReport(['main/loss', 'validation/main/loss'], 'epoch', file_name='loss_{}.png'.format(trial.number)))
# Write a log of evaluation statistics for each epoch
trainer.extend(extensions.PlotReport(['main/accuracy', 'validation/main/accuracy'], 'epoch', file_name='accuracy_{}.png'.format(trial.number)))
# Run the training
trainer.run()
# save the model
chainer.serializers.save_npz('./result/hypara/{0}.model'.format(trial.number), model)
# return the AUC
graphs, target = dataset.converter(test, device)
with chainer.using_config('train', False), chainer.using_config('enable_backprop', False):
y_pred = model.predictor(graphs)
y_pred.to_cpu()
y_pred = y_pred.array
target = chainer.cuda.to_cpu(target)
return 1 - sklearn.metrics.roc_auc_score(target, y_pred[:,1])
def __init__(self,
num_entities,
num_node_features,
num_edge_features,
config=None,
entity_names=None,
n_regions=2):
self.n_regions = n_regions
GNN.__init__(self, num_entities, num_node_features, num_edge_features,
config, entity_names)
self.num_entities = num_entities
self.activation = 'relu'
self.config = config
self.top_k = config.top_k
self.weight_file_name = self.create_weight_file_name()
self.optimizer = self.create_optimizer(config.optimizer, config.lr)
self.num_node_features = num_node_features
self.node_input, self.edge_input, self.action_input, self.op_input, self.cost_input = \
self.create_inputs(num_entities, num_node_features, num_edge_features, dim_action=[1])
self.n_node_features = num_node_features
def test(test_dataloader, model=None):
if model is None:
obj = torch.load(args.save_path + '.model',
map_location=lambda storage, loc: storage)
train_args = obj['args']
model = GNN(word_vocab_size=WORD_VOCAB_SIZE,
char_vocab_size=CHAR_VOCAB_SIZE,
d_output=d_output,
args=train_args)
model.load_state_dict(obj['model'])
model.cuda()
print('Model loaded.')
test_acc, test_prec, test_recall, test_f1 = evaluate(model,
test_dataloader,
output=True,
args=args)
print('######## prec : ', acc_to_str(test_prec))
print('######## recall : ', acc_to_str(test_recall))
print('######## f1 : ', acc_to_str(test_f1))
prec, recall, f1 = np.mean(list(test_prec.values())), np.mean(
list(test_recall.values())), np.mean(list(test_f1.values()))
print(prec, recall, f1)
result_obj['test_prec'] = prec
result_obj['test_recall'] = recall
result_obj['test_f1'] = f1
result_obj['test_info'] = '\n'.join(
[acc_to_str(test_prec),
acc_to_str(test_recall),
acc_to_str(test_f1)])
def main():
train_data = read_train()
random.shuffle(train_data)
test = train_data[1500:]
train_data = train_data[:1500]
ini = make_initial()
gnn = GNN(15, 8, ini['x'])
EPOCHS = 100
dmsgd = momentum_sgd(gnn, 50, train_data, 0.001, ini["param"], 2, EPOCHS,
0.9, test)
dsgd = sgd(gnn, 50, train_data, 0.001, ini["param"], 2, EPOCHS, test)
with open('sgd.pickle', 'wb') as handle:
pickle.dump(dsgd, handle, protocol=pickle.HIGHEST_PROTOCOL)
with open('msgd.pickle', 'wb') as handle:
pickle.dump(dmsgd, handle, protocol=pickle.HIGHEST_PROTOCOL)
with open('sgd.pickle', 'rb') as handle:
dsgd = pickle.load(handle)
with open('msgd.pickle', 'rb') as handle:
dmsgd = pickle.load(handle)
plot(dsgd, dmsgd)
def approach_1(args):
device = chainer.get_device(args.device)
# load the trained model, model attributes and graph dataset
with open('{}_stat'.format(args.model_name), mode='rb') as f:
model_args = pickle.load(f)
dataset = util.GraphData("mixed", model_args.feat_init_method, "Regression", args.data_num, device)
model = chainer.links.Classifier(GNN(model_args.num_layers, model_args.num_mlp_layers, dataset.graphs[0].node_features.shape[1],
model_args.hidden_dim, 2, model_args.final_dropout, model_args.graph_pooling_type,
model_args.neighbor_pooling_type, model_args.task_type))
chainer.serializers.load_npz(args.model_name, model)
model.to_device(device)
device.use()
print('\n--- Prediction by existing algorithm ---')
print('calctype\tIndex\t|V|\t|E|\ttw(G)\ttime\tevaltw\tfunccallnum')
output_file = "exist.dat"
result = ["ID\t|V|\t|E|\ttw\ttime\tevaltw\tfunccallnum"]
for calc in ["upper", "lower"]:
for idx in range(0, len(dataset.graphs)):
eval_graph = dataset.graphs[idx]
graphstat = "{4}\t{3}\t{0}\t{1}\t{2}".format(eval_graph.g.number_of_nodes(), eval_graph.g.number_of_edges(), dataset.labels[idx], str(idx).rjust(5), calc)
print(graphstat, end='\t')
try:
tm, evtw, fcn = ordinaryDP(eval_graph, calc)
res = "{0}\t{1}\t{2}".format(tm, evtw, fcn)
except timeout_decorator.TimeoutError:
res = "TimeOut"
print(res)
result.append(graphstat + "\t" + res)
# write results to a file
if args.out_to_file:
with open("./{}/Approach1/".format(args.out) + output_file, "w") as f:
f.write('\n'.join(result))
def readout():
N = 4 # 頂点の数
D = 4 # dimension
edge = [[1, 2], [2, 3], [2, 4], [3, 4]] # Edge
x = np.array([[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]])
W = np.array([
[0.5, 0.3, 0, 0],
[0.2, 0.1, 0, -0.1],
[-0.4, -0.5, 1, 0],
[-3, 0, 0, 1],
])
g = GNN(N, D, x)
a = g.aggregate_1(x, g.get_adjacency_matrix(edge))
# print(a)
# print(W @ a)
nx = g.aggregate_2(W, a)
# print(nx)
h = g.readout(nx)
return h
def main():
# Training settings
parser = argparse.ArgumentParser(
description='GNN baselines on pcqm4m with DGL')
parser.add_argument('--seed',
type=int,
default=42,
help='random seed to use (default: 42)')
parser.add_argument('--device',
type=int,
default=0,
help='which gpu to use if any (default: 0)')
parser.add_argument(
'--gnn',
type=str,
default='gin-virtual',
help='GNN to use, which can be from '
'[gin, gin-virtual, gcn, gcn-virtual] (default: gin-virtual)')
parser.add_argument(
'--graph_pooling',
type=str,
default='sum',
help='graph pooling strategy mean or sum (default: sum)')
parser.add_argument('--drop_ratio',
type=float,
default=0,
help='dropout ratio (default: 0)')
parser.add_argument(
'--num_layers',
type=int,
default=5,
help='number of GNN message passing layers (default: 5)')
parser.add_argument(
'--emb_dim',
type=int,
default=600,
help='dimensionality of hidden units in GNNs (default: 600)')
parser.add_argument('--batch_size',
type=int,
default=256,
help='input batch size for training (default: 256)')
parser.add_argument('--num_workers',
type=int,
default=0,
help='number of workers (default: 0)')
parser.add_argument('--checkpoint_dir',
type=str,
default='',
help='directory to save checkpoint')
parser.add_argument('--save_test_dir',
type=str,
default='',
help='directory to save test submission file')
args = parser.parse_args()
print(args)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
random.seed(args.seed)
if torch.cuda.is_available():
torch.cuda.manual_seed(args.seed)
device = torch.device("cuda:" + str(args.device))
else:
device = torch.device("cpu")
### automatic data loading and splitting
### Read in the raw SMILES strings
smiles_dataset = PCQM4MDataset(root='dataset/', only_smiles=True)
split_idx = smiles_dataset.get_idx_split()
test_smiles_dataset = [smiles_dataset[i] for i in split_idx['test']]
onthefly_dataset = OnTheFlyPCQMDataset(test_smiles_dataset)
test_loader = DataLoader(onthefly_dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.num_workers,
collate_fn=collate_dgl)
### automatic evaluator.
evaluator = PCQM4MEvaluator()
shared_params = {
'num_layers': args.num_layers,
'emb_dim': args.emb_dim,
'drop_ratio': args.drop_ratio,
'graph_pooling': args.graph_pooling
}
if args.gnn == 'gin':
model = GNN(gnn_type='gin', virtual_node=False,
**shared_params).to(device)
elif args.gnn == 'gin-virtual':
model = GNN(gnn_type='gin', virtual_node=True,
**shared_params).to(device)
elif args.gnn == 'gcn':
model = GNN(gnn_type='gcn', virtual_node=False,
**shared_params).to(device)
elif args.gnn == 'gcn-virtual':
model = GNN(gnn_type='gcn', virtual_node=True,
**shared_params).to(device)
else:
raise ValueError('Invalid GNN type')
num_params = sum(p.numel() for p in model.parameters())
print(f'#Params: {num_params}')
checkpoint_path = os.path.join(args.checkpoint_dir, 'checkpoint.pt')
if not os.path.exists(checkpoint_path):
raise RuntimeError(f'Checkpoint file not found at {checkpoint_path}')
## reading in checkpoint
checkpoint = torch.load(checkpoint_path)
model.load_state_dict(checkpoint['model_state_dict'])
print('Predicting on test data...')
y_pred = test(model, device, test_loader)
print('Saving test submission file...')
evaluator.save_test_submission({'y_pred': y_pred}, args.save_test_dir)
max_it = 50
num_epoch = 5000
optimizer = tf.train.AdamOptimizer
# initialize state and output network
net = n.Net(input_dim, state_dim, output_dim)
# initialize GNN
param = "st_d" + str(state_dim) + "_th" + str(threshold) + "_lr" + str(
learning_rate)
print(param)
g = GNN.GNN(net,
max_it=max_it,
input_dim=input_dim,
output_dim=output_dim,
state_dim=state_dim,
optimizer=optimizer,
learning_rate=learning_rate,
threshold=threshold,
param=param,
config=config)
count = 0
# train the model and validate every 30 epochs
for j in range(0, num_epoch):
g.Train(inp[0], arcnode[0], labels, count, nodegraph[0])
if count % 30 == 0:
print("Epoch ", count)
print("Training: ",
g.Validate(inp[0], arcnode[0], labels, count, nodegraph[0]))
print(
def main():
device = torch.device("cuda:" + str(args.device)) if torch.cuda.is_available() else torch.device("cpu")
### automatic dataloading and splitting
dataset = PygGraphPropPredDataset(name = args.dataset, root='/cmlscratch/kong/datasets/ogb')
seq_len_list = np.array([len(seq) for seq in dataset.data.y])
print('Target seqence less or equal to {} is {}%.'.format(args.max_seq_len, np.sum(seq_len_list <= args.max_seq_len) / len(seq_len_list)))
split_idx = dataset.get_idx_split()
# print(split_idx['train'])
# print(split_idx['valid'])
# print(split_idx['test'])
# train_method_name = [' '.join(dataset.data.y[i]) for i in split_idx['train']]
# valid_method_name = [' '.join(dataset.data.y[i]) for i in split_idx['valid']]
# test_method_name = [' '.join(dataset.data.y[i]) for i in split_idx['test']]
# print('#train')
# print(len(train_method_name))
# print('#valid')
# print(len(valid_method_name))
# print('#test')
# print(len(test_method_name))
# train_method_name_set = set(train_method_name)
# valid_method_name_set = set(valid_method_name)
# test_method_name_set = set(test_method_name)
# # unique method name
# print('#unique train')
# print(len(train_method_name_set))
# print('#unique valid')
# print(len(valid_method_name_set))
# print('#unique test')
# print(len(test_method_name_set))
# # unique valid/test method name
# print('#valid unseen during training')
# print(len(valid_method_name_set - train_method_name_set))
# print('#test unseen during training')
# print(len(test_method_name_set - train_method_name_set))
### building vocabulary for sequence predition. Only use training data.
vocab2idx, idx2vocab = get_vocab_mapping([dataset.data.y[i] for i in split_idx['train']], args.num_vocab)
# test encoder and decoder
# for data in dataset:
# # PyG >= 1.5.0
# print(data.y)
#
# # PyG 1.4.3
# # print(data.y[0])
# data = encode_y_to_arr(data, vocab2idx, args.max_seq_len)
# print(data.y_arr[0])
# decoded_seq = decode_arr_to_seq(data.y_arr[0], idx2vocab)
# print(decoded_seq)
# print('')
## test augment_edge
# data = dataset[2]
# print(data)
# data_augmented = augment_edge(data)
# print(data_augmented)
### set the transform function
# augment_edge: add next-token edge as well as inverse edges. add edge attributes.
# encode_y_to_arr: add y_arr to PyG data object, indicating the array representation of a sequence.
dataset.transform = transforms.Compose([augment_edge, lambda data: encode_y_to_arr(data, vocab2idx, args.max_seq_len)])
### automatic evaluator. takes dataset name as input
evaluator = Evaluator(args.dataset)
train_loader = DataLoader(dataset[split_idx["train"]], batch_size=args.batch_size, shuffle=True, num_workers = args.num_workers)
valid_loader = DataLoader(dataset[split_idx["valid"]], batch_size=args.batch_size, shuffle=False, num_workers = args.num_workers)
test_loader = DataLoader(dataset[split_idx["test"]], batch_size=args.batch_size, shuffle=False, num_workers = args.num_workers)
nodetypes_mapping = pd.read_csv(os.path.join(dataset.root, 'mapping', 'typeidx2type.csv.gz'))
nodeattributes_mapping = pd.read_csv(os.path.join(dataset.root, 'mapping', 'attridx2attr.csv.gz'))
### Encoding node features into emb_dim vectors.
### The following three node features are used.
# 1. node type
# 2. node attribute
# 3. node depth
node_encoder = ASTNodeEncoder(args.emb_dim, num_nodetypes = len(nodetypes_mapping['type']), num_nodeattributes = len(nodeattributes_mapping['attr']), max_depth = 20)
vals, tests = [], []
for run in range(args.runs):
best_val, final_test = 0, 0
if args.gnn == 'gin':
model = GNN(num_vocab=len(vocab2idx), max_seq_len=args.max_seq_len, node_encoder=node_encoder,
num_layer=args.num_layer, gnn_type='gin', emb_dim=args.emb_dim, drop_ratio=args.drop_ratio,
virtual_node=False).to(device)
elif args.gnn == 'gin-virtual':
model = GNN(num_vocab=len(vocab2idx), max_seq_len=args.max_seq_len, node_encoder=node_encoder,
num_layer=args.num_layer, gnn_type='gin', emb_dim=args.emb_dim, drop_ratio=args.drop_ratio,
virtual_node=True).to(device)
elif args.gnn == 'gcn':
model = GNN(num_vocab=len(vocab2idx), max_seq_len=args.max_seq_len, node_encoder=node_encoder,
num_layer=args.num_layer, gnn_type='gcn', emb_dim=args.emb_dim, drop_ratio=args.drop_ratio,
virtual_node=False).to(device)
elif args.gnn == 'gcn-virtual':
model = GNN(num_vocab=len(vocab2idx), max_seq_len=args.max_seq_len, node_encoder=node_encoder,
num_layer=args.num_layer, gnn_type='gcn', emb_dim=args.emb_dim, drop_ratio=args.drop_ratio,
virtual_node=True).to(device)
else:
raise ValueError('Invalid GNN type')
optimizer = torch.optim.Adam(model.parameters(), lr=args.lr)
for epoch in range(1, args.epochs+1):
loss = train(model, device, train_loader, optimizer, args)
if epoch > args.epochs // 2 and epoch % args.test_freq == 0 or epoch == args.epochs:
#4min
train_perf = eval(model, device, train_loader, evaluator,
arr_to_seq=lambda arr: decode_arr_to_seq(arr, idx2vocab))
valid_perf = eval(model, device, valid_loader, evaluator,
arr_to_seq=lambda arr: decode_arr_to_seq(arr, idx2vocab))
test_perf = eval(model, device, test_loader, evaluator,
arr_to_seq=lambda arr: decode_arr_to_seq(arr, idx2vocab))
result = (train_perf[dataset.eval_metric], valid_perf[dataset.eval_metric], test_perf[dataset.eval_metric])
_, val, tst = result
if val > best_val:
best_val = val
final_test = tst
print(f'Run{run} val:{best_val}, test:{final_test}')
vals.append(best_val)
tests.append(final_test)
print('')
print(f"Average val accuracy: {np.mean(vals)} ± {np.std(vals)}")
print(f"Average test accuracy: {np.mean(tests)} ± {np.std(tests)}")
target_p = torch.zeros(opt['num_node'], opt['num_class'])
if opt['cuda']:
inputs = inputs.cuda()
target = target.cuda()
idx_train = idx_train.cuda()
idx_dev = idx_dev.cuda()
idx_test = idx_test.cuda()
idx_all = idx_all.cuda()
idx_unlabeled = idx_unlabeled.cuda()
inputs_q = inputs_q.cuda()
target_q = target_q.cuda()
inputs_p = inputs_p.cuda()
target_p = target_p.cuda()
gnn = GNN(opt, adj)
trainer = Trainer(opt, gnn)
# Build the ema model
gnn_ema = GNN(opt, adj)
for ema_param, param in zip(gnn_ema.parameters(), gnn.parameters()):
ema_param.data= param.data
for param in gnn_ema.parameters():
param.detach_()
trainer_ema = Trainer(opt, gnn_ema, ema = False)
def init_data():
inputs_q.copy_(inputs)
def main():
# Training settings
parser = argparse.ArgumentParser(
description="GNN baselines on pcqm4m with Pytorch Geometrics")
parser.add_argument("--device",
type=int,
default=0,
help="which gpu to use if any (default: 0)")
parser.add_argument(
"--gnn",
type=str,
default="gin-virtual",
help=
"GNN gin, gin-virtual, or gcn, or gcn-virtual (default: gin-virtual)",
)
parser.add_argument(
"--graph_pooling",
type=str,
default="sum",
help="graph pooling strategy mean or sum (default: sum)",
)
parser.add_argument("--drop_ratio",
type=float,
default=0,
help="dropout ratio (default: 0)")
parser.add_argument(
"--num_layers",
type=int,
default=5,
help="number of GNN message passing layers (default: 5)",
)
parser.add_argument(
"--emb_dim",
type=int,
default=600,
help="dimensionality of hidden units in GNNs (default: 600)",
)
parser.add_argument("--train_subset", action="store_true")
parser.add_argument(
"--batch_size",
type=int,
default=256,
help="input batch size for training (default: 256)",
)
parser.add_argument(
"--epochs",
type=int,
default=100,
help="number of epochs to train (default: 100)",
)
parser.add_argument("--num_workers",
type=int,
default=0,
help="number of workers (default: 0)")
parser.add_argument("--log_dir",
type=str,
default="",
help="tensorboard log directory")
parser.add_argument("--checkpoint_dir",
type=str,
default="",
help="directory to save checkpoint")
parser.add_argument(
"--save_test_dir",
type=str,
default="",
help="directory to save test submission file",
)
args = parser.parse_args()
print(args)
np.random.seed(42)
torch.manual_seed(42)
torch.cuda.manual_seed(42)
random.seed(42)
device = (torch.device("cuda:" + str(args.device))
if torch.cuda.is_available() else torch.device("cpu"))
### automatic dataloading and splitting
dataset = PygPCQM4MDataset(root="dataset/")
split_idx = dataset.get_idx_split()
### automatic evaluator. takes dataset name as input
evaluator = PCQM4MEvaluator()
if args.train_subset:
subset_ratio = 0.1
subset_idx = torch.randperm(len(
split_idx["train"]))[:int(subset_ratio * len(split_idx["train"]))]
train_loader = DataLoader(
dataset[split_idx["train"][subset_idx]],
batch_size=args.batch_size,
shuffle=True,
num_workers=args.num_workers,
)
else:
train_loader = DataLoader(
dataset[split_idx["train"]],
batch_size=args.batch_size,
shuffle=True,
num_workers=args.num_workers,
)
valid_loader = DataLoader(
dataset[split_idx["valid"]],
batch_size=args.batch_size,
shuffle=False,
num_workers=args.num_workers,
)
if args.save_test_dir is not "":
test_loader = DataLoader(
dataset[split_idx["test"]],
batch_size=args.batch_size,
shuffle=False,
num_workers=args.num_workers,
)
if args.checkpoint_dir is not "":
os.makedirs(args.checkpoint_dir, exist_ok=True)
shared_params = {
"num_layers": args.num_layers,
"emb_dim": args.emb_dim,
"drop_ratio": args.drop_ratio,
"graph_pooling": args.graph_pooling,
}
if args.gnn == "gin":
model = GNN(gnn_type="gin", virtual_node=False,
**shared_params).to(device)
elif args.gnn == "gin-virtual":
model = GNN(gnn_type="gin", virtual_node=True,
**shared_params).to(device)
elif args.gnn == "gcn":
model = GNN(gnn_type="gcn", virtual_node=False,
**shared_params).to(device)
elif args.gnn == "gcn-virtual":
model = GNN(gnn_type="gcn", virtual_node=True,
**shared_params).to(device)
else:
raise ValueError("Invalid GNN type")
num_params = sum(p.numel() for p in model.parameters())
print(f"#Params: {num_params}")
optimizer = optim.Adam(model.parameters(), lr=0.001)
if args.log_dir is not "":
writer = SummaryWriter(log_dir=args.log_dir)
best_valid_mae = 1000
if args.train_subset:
scheduler = StepLR(optimizer, step_size=300, gamma=0.25)
args.epochs = 1000
else:
scheduler = StepLR(optimizer, step_size=30, gamma=0.25)
for epoch in range(1, args.epochs + 1):
print("=====Epoch {}".format(epoch))
print("Training...")
train_mae = train(model, device, train_loader, optimizer)
print("Evaluating...")
valid_mae = eval(model, device, valid_loader, evaluator)
print({"Train": train_mae, "Validation": valid_mae})
if args.log_dir is not "":
writer.add_scalar("valid/mae", valid_mae, epoch)
writer.add_scalar("train/mae", train_mae, epoch)
if valid_mae < best_valid_mae:
best_valid_mae = valid_mae
if args.checkpoint_dir is not "":
print("Saving checkpoint...")
checkpoint = {
"epoch": epoch,
"model_state_dict": model.state_dict(),
"optimizer_state_dict": optimizer.state_dict(),
"scheduler_state_dict": scheduler.state_dict(),
"best_val_mae": best_valid_mae,
"num_params": num_params,
}
torch.save(checkpoint,
os.path.join(args.checkpoint_dir, "checkpoint.pt"))
if args.save_test_dir is not "":
print("Predicting on test data...")
y_pred = test(model, device, test_loader)
print("Saving test submission file...")
evaluator.save_test_submission({"y_pred": y_pred},
args.save_test_dir)
scheduler.step()
print(f"Best validation MAE so far: {best_valid_mae}")
if args.log_dir is not "":
writer.close()
input_dim = inp.shape[1]
output_dim = labels.shape[1]
max_it = 50
num_epoch = 10000
optimizer = tf.train.AdamOptimizer
# initialize state and output network
net = n.Net(input_dim, state_dim, output_dim)
# initialize GNN
param = "st_d" + str(state_dim) + "_th" + str(threshold) + "_lr" + str(learning_rate)
print(param)
tensorboard = False
g = GNN.GNN(net, input_dim, output_dim, state_dim, max_it, optimizer, learning_rate, threshold, graph_based=False, param=param, config=config,
tensorboard=tensorboard)
# train the model
count = 0
######
for j in range(0, num_epoch):
_, it = g.Train(inputs=inp, ArcNode=arcnode, target=labels, step=count)
#if count % 30 == 0:
if count % 3000 == 0:
print("Epoch ", count)
print("Training: ", g.Validate(inp, arcnode, labels, count))
# end = time.time()
def main():
seed = args.seed
np.random.seed(seed)
random.seed(seed)
torch.manual_seed(seed)
torch.cuda.manual_seed(seed)
torch.cuda.manual_seed_all(seed)
model_save_dir = f'models/{args.name}'
os.makedirs(model_save_dir, exist_ok=True)
device = torch.device(
"cuda:" +
str(args.device)) if torch.cuda.is_available() else torch.device("cpu")
print("Training")
# writer = SummaryWriter(model_save_dir)
with open(f'{model_save_dir}/arguments.txt', 'w') as f:
json.dump(args.__dict__, f, indent=2)
### automatic dataloading and splitting
dataset = PygGraphPropPredDataset(name=args.dataset, transform=add_zeros)
split_idx = dataset.get_idx_split()
### automatic evaluator. takes dataset name as input
evaluator = Evaluator(args.dataset)
train_loader = DataLoader(dataset[split_idx["train"]],
batch_size=args.batch_size,
shuffle=True,
num_workers=args.num_workers)
valid_loader = DataLoader(dataset[split_idx["valid"]],
batch_size=args.batch_size,
shuffle=False,
num_workers=args.num_workers)
test_loader = DataLoader(dataset[split_idx["test"]],
batch_size=args.batch_size,
shuffle=False,
num_workers=args.num_workers)
vals, tests = [], []
for run in range(args.runs):
best_val, final_test = 0, 0
if args.gnn == 'gin':
model = GNN(gnn_type='gin',
num_class=dataset.num_classes,
num_layer=args.num_layer,
emb_dim=args.emb_dim,
drop_ratio=args.drop_ratio,
virtual_node=False,
topological=args.topological).to(device)
elif args.gnn == 'gin-virtual':
model = GNN(gnn_type='gin',
num_class=dataset.num_classes,
num_layer=args.num_layer,
emb_dim=args.emb_dim,
drop_ratio=args.drop_ratio,
virtual_node=True,
topological=args.topological).to(device)
elif args.gnn == 'gcn':
model = GNN(gnn_type='gcn',
num_class=dataset.num_classes,
num_layer=args.num_layer,
emb_dim=args.emb_dim,
drop_ratio=args.drop_ratio,
virtual_node=False,
topological=args.topological).to(device)
elif args.gnn == 'gcn-virtual':
model = GNN(gnn_type='gcn',
num_class=dataset.num_classes,
num_layer=args.num_layer,
emb_dim=args.emb_dim,
drop_ratio=args.drop_ratio,
virtual_node=True,
topological=args.topological).to(device)
elif args.gnn == 'controller':
model = ControllerTransformer().to(device)
else:
raise ValueError('Invalid GNN type')
optimizer = torch.optim.Adam(model.parameters(), lr=args.lr)
for epoch in range(1, args.epochs + 1):
loss = train(model, device, train_loader, optimizer, args)
if epoch > args.epochs // 2 and epoch % args.test_freq == 0 or epoch == args.epochs:
# 4min
train_perf = eval(model, device, train_loader, evaluator)
valid_perf = eval(model, device, valid_loader, evaluator)
test_perf = eval(model, device, test_loader, evaluator)
result = (train_perf[dataset.eval_metric],
valid_perf[dataset.eval_metric],
test_perf[dataset.eval_metric])
_, val, tst = result
if val > best_val:
torch.save(model.state_dict(),
os.path.join(model_save_dir, f'model-best.pth'))
best_val = val
final_test = tst
print(f'Run{run} val:{best_val}, test:{final_test}')
vals.append(best_val)
tests.append(final_test)
print('')
print(f"Average val accuracy: {np.mean(vals)} ± {np.std(vals)}")
print(f"Average test accuracy: {np.mean(tests)} ± {np.std(tests)}")
def main():
############# training set ##########
set_name = "cli_15_7_200"
############# training set ################
#inp, arcnode, nodegraph, nodein, labels = Library.set_load_subgraph(data_path, "train")
inp, arcnode, nodegraph, nodein, labels, _ = gnn_utils.set_load_general(
data_path, "train", set_name=set_name)
############ test set ####################
#inp_test, arcnode_test, nodegraph_test, nodein_test, labels_test = Library.set_load_subgraph(data_path, "test")
inp_test, arcnode_test, nodegraph_test, nodein_test, labels_test, _ = gnn_utils.set_load_general(
data_path, "test", set_name=set_name)
############ validation set #############
#inp_val, arcnode_val, nodegraph_val, nodein_val, labels_val = Library.set_load_subgraph(data_path, "valid")
inp_val, arcnode_val, nodegraph_val, nodein_val, labels_val, _ = gnn_utils.set_load_general(
data_path, "validation", set_name=set_name)
# set threshold, learning rate and state dimension
threshold = 0.001
learning_rate = 0.01
state_dim = 5
# set input and output dim, the maximum number of iterations, the number of epochs and the optimizer
tf.reset_default_graph()
input_dim = len(inp[0][0])
output_dim = 2
max_it = 50
num_epoch = 5000
optimizer = tf.train.AdamOptimizer
# initialize state and output network
net = n.Net(input_dim, state_dim, output_dim)
# initialize GNN
param = "st_d" + str(state_dim) + "_th" + str(threshold) + "_lr" + str(
learning_rate)
print(param)
g = GNN.GNN(net,
max_it=max_it,
input_dim=input_dim,
output_dim=output_dim,
state_dim=state_dim,
optimizer=optimizer,
learning_rate=learning_rate,
threshold=threshold,
param=param,
config=config)
count = 0
# train the model and validate every 30 epochs
for j in range(0, num_epoch):
g.Train(inp[0], arcnode[0], labels, count, nodegraph[0])
if count % 30 == 0:
print("Epoch ", count)
print("Training: ",
g.Validate(inp[0], arcnode[0], labels, count, nodegraph[0]))
print(
"Validation: ",
g.Validate(inp_val[0], arcnode_val[0], labels_val, count,
nodegraph_val[0]))
count = count + 1
# evaluate on the test set
print(
g.Evaluate(inp_test[0], arcnode_test[0], labels_test,
nodegraph_test[0]))
def main():
# Training settings
parser = argparse.ArgumentParser(
description='GNN baselines on pcqm4m with DGL')
parser.add_argument('--seed',
type=int,
default=42,
help='random seed to use (default: 42)')
parser.add_argument('--device',
type=int,
default=0,
help='which gpu to use if any (default: 0)')
parser.add_argument(
'--gnn',
type=str,
default='gin-virtual',
help='GNN to use, which can be from '
'[gin, gin-virtual, gcn, gcn-virtual] (default: gin-virtual)')
parser.add_argument(
'--graph_pooling',
type=str,
default='sum',
help='graph pooling strategy mean or sum (default: sum)')
parser.add_argument('--drop_ratio',
type=float,
default=0,
help='dropout ratio (default: 0)')
parser.add_argument(
'--num_layers',
type=int,
default=5,
help='number of GNN message passing layers (default: 5)')
parser.add_argument(
'--emb_dim',
type=int,
default=600,
help='dimensionality of hidden units in GNNs (default: 600)')
parser.add_argument('--train_subset',
action='store_true',
help='use 10% of the training set for training')
parser.add_argument('--batch_size',
type=int,
default=256,
help='input batch size for training (default: 256)')
parser.add_argument('--epochs',
type=int,
default=100,
help='number of epochs to train (default: 100)')
parser.add_argument('--num_workers',
type=int,
default=0,
help='number of workers (default: 0)')
parser.add_argument('--log_dir',
type=str,
default="",
help='tensorboard log directory. If not specified, '
'tensorboard will not be used.')
parser.add_argument('--checkpoint_dir',
type=str,
default='',
help='directory to save checkpoint')
parser.add_argument('--save_test_dir',
type=str,
default='',
help='directory to save test submission file')
args = parser.parse_args()
print(args)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
random.seed(args.seed)
if torch.cuda.is_available():
torch.cuda.manual_seed(args.seed)
device = torch.device("cuda:" + str(args.device))
else:
device = torch.device("cpu")
### automatic dataloading and splitting
dataset = SampleDglPCQM4MDataset(root='dataset/')
# split_idx['train'], split_idx['valid'], split_idx['test']
# separately gives a 1D int64 tensor
split_idx = dataset.get_idx_split()
split_idx["train"] = split_idx["train"].type(torch.LongTensor)
split_idx["test"] = split_idx["test"].type(torch.LongTensor)
split_idx["valid"] = split_idx["valid"].type(torch.LongTensor)
### automatic evaluator.
evaluator = PCQM4MEvaluator()
if args.train_subset:
subset_ratio = 0.1
subset_idx = torch.randperm(len(
split_idx["train"]))[:int(subset_ratio * len(split_idx["train"]))]
train_loader = DataLoader(dataset[split_idx["train"][subset_idx]],
batch_size=args.batch_size,
shuffle=True,
num_workers=args.num_workers,
collate_fn=collate_dgl)
else:
train_loader = DataLoader(dataset[split_idx["train"]],
batch_size=args.batch_size,
shuffle=True,
num_workers=args.num_workers,
collate_fn=collate_dgl)
valid_loader = DataLoader(dataset[split_idx["valid"]],
batch_size=args.batch_size,
shuffle=False,
num_workers=args.num_workers,
collate_fn=collate_dgl)
if args.save_test_dir is not '':
test_loader = DataLoader(dataset[split_idx["test"]],
batch_size=args.batch_size,
shuffle=False,
num_workers=args.num_workers,
collate_fn=collate_dgl)
if args.checkpoint_dir is not '':
os.makedirs(args.checkpoint_dir, exist_ok=True)
shared_params = {
'num_layers': args.num_layers,
'emb_dim': args.emb_dim,
'drop_ratio': args.drop_ratio,
'graph_pooling': args.graph_pooling
}
if args.gnn == 'gin':
model = GNN(gnn_type='gin', virtual_node=False,
**shared_params).to(device)
elif args.gnn == 'gin-virtual':
model = GNN(gnn_type='gin', virtual_node=True,
**shared_params).to(device)
elif args.gnn == 'gcn':
model = GNN(gnn_type='gcn', virtual_node=False,
**shared_params).to(device)
elif args.gnn == 'gcn-virtual':
model = GNN(gnn_type='gcn', virtual_node=True,
**shared_params).to(device)
elif args.gnn == 'gin-virtual-diffpool':
model = DiffPoolGNN(gnn_type='gin', virtual_node=True,
**shared_params).to(device)
elif args.gnn == 'gin-virtual-bayes-diffpool':
model = BayesDiffPoolGNN(gnn_type='gin',
virtual_node=True,
**shared_params).to(device)
else:
raise ValueError('Invalid GNN type')
num_params = sum(p.numel() for p in model.parameters())
print(f'#Params: {num_params}')
optimizer = optim.Adam(model.parameters(), lr=0.001)
if args.log_dir is not '':
writer = SummaryWriter(log_dir=args.log_dir)
best_valid_mae = 1000
if args.train_subset:
scheduler = StepLR(optimizer, step_size=300, gamma=0.25)
args.epochs = 1000
else:
scheduler = StepLR(optimizer, step_size=30, gamma=0.25)
""" load from latest checkpoint """
# start epoch (default = 1, unless resuming training)
firstEpoch = 1
# check if checkpoint exist -> load model
checkpointFile = os.path.join(args.checkpoint_dir, 'checkpoint.pt')
if os.path.exists(checkpointFile):
# load checkpoint file
checkpointData = torch.load(checkpointFile)
firstEpoch = checkpointData["epoch"]
model.load_state_dict(checkpointData["model_state_dict"])
optimizer.load_state_dict(checkpointData["optimizer_state_dict"])
scheduler.load_state_dict(checkpointData["scheduler_state_dict"])
best_valid_mae = checkpointData["best_val_mae"]
num_params = checkpointData["num_params"]
print(
"Loaded existing weights from {}. Continuing from epoch: {} with best valid MAE: {}"
.format(checkpointFile, firstEpoch, best_valid_mae))
for epoch in range(firstEpoch, args.epochs + 1):
print("=====Epoch {}".format(epoch))
print('Training...')
train_mae = train(model, device, train_loader, optimizer, args.gnn)
print('Evaluating...')
valid_mae = eval(model, device, valid_loader, evaluator)
print({'Train': train_mae, 'Validation': valid_mae})
if args.log_dir is not '':
writer.add_scalar('valid/mae', valid_mae, epoch)
writer.add_scalar('train/mae', train_mae, epoch)
if valid_mae < best_valid_mae:
best_valid_mae = valid_mae
if args.checkpoint_dir is not '':
print('Saving checkpoint...')
checkpoint = {
'epoch': epoch,
'model_state_dict': model.state_dict(),
'optimizer_state_dict': optimizer.state_dict(),
'scheduler_state_dict': scheduler.state_dict(),
'best_val_mae': best_valid_mae,
'num_params': num_params
}
torch.save(checkpoint,
os.path.join(args.checkpoint_dir, 'checkpoint.pt'))
if args.save_test_dir is not '':
print('Predicting on test data...')
y_pred = test(model, device, test_loader)
print('Saving test submission file...')
evaluator.save_test_submission({'y_pred': y_pred},
args.save_test_dir)
scheduler.step()
print(f'Best validation MAE so far: {best_valid_mae}')
if args.log_dir is not '':
writer.close()
'''
PFN internship 2019 coding task
machine learning
task-1
Issei NAKASONE
'''
import datasets as D
from gnn import GNN
filepath = '../datasets/train/0_graph.txt'
graph = D.read_graph(filepath)
model = GNN()
h_G, _ = model.forward(graph, T=2)
print(h_G)
'''
PFN internship 2019 coding task
machine learning
task-3
Issei NAKASONE
'''
import datasets as D
import optimizers as op
from gnn import GNN, TrainGNN
from iterator import Iterator
dirpath = '../datasets/train/'
batch_size = 128
train, test = D.get_dataset(dirpath, test_ratio=0.25)
train_iter = Iterator(train, batch_size)
test_iter = Iterator(test, batch_size)
model = GNN()
optimizer = op.SGD()
#optimizer = op.MomentumSGD()
optimizer.setup(model)
trainer = TrainGNN(optimizer, train_iter, test_iter)
trainer.start(epoch=50)
def train(seed):
print('random seed:', seed)
torch.manual_seed(seed)
torch.cuda.manual_seed(seed)
# torch.backends.cudnn.enabled = False
dataset = read_data('../data', '../graph')
label2id = dataset.label2id
print(label2id)
vocab_size = dataset.vocab_size
output_dim = len(label2id)
def acc_to_str(acc):
s = ['%s:%.3f' % (label, acc[label]) for label in acc]
return '{' + ', '.join(s) + '}'
cross_res = {label: [] for label in label2id if label != 'O'}
output_file = open('%s.mistakes' % args.output, 'w')
for cross_valid in range(5):
model = GNN(vocab_size=vocab_size, output_dim=output_dim, args=args)
model.cuda()
# print vocab_size
dataset.split_train_valid_test([0.8, 0.1, 0.1], 5, cross_valid)
print('train:', len(dataset.train), 'valid:', len(dataset.valid),
'test:', len(dataset.test))
def evaluate(model, datalist, output_file=None):
if output_file != None:
output_file.write(
'#############################################\n')
correct = {label: 0 for label in label2id if label != 'O'}
total = len(datalist)
model.eval()
print_cnt = 0
for data in datalist:
word, feat = Variable(data.input_word).cuda(), Variable(
data.input_feat).cuda()
a_ud, a_lr = Variable(data.a_ud,
requires_grad=False).cuda(), Variable(
data.a_lr,
requires_grad=False).cuda()
mask = Variable(data.mask, requires_grad=False).cuda()
if args.globalnode:
logprob, form = model(word, feat, mask, a_ud, a_lr)
logprob = logprob.data.view(-1, output_dim)
else:
logprob = model(word, feat, mask, a_ud,
a_lr).data.view(-1, output_dim)
mask = mask.data.view(-1)
y_pred = torch.LongTensor(output_dim)
for i in range(output_dim):
prob = logprob[:, i].exp() * mask
y_pred[i] = prob.topk(k=1)[1][0]
# y_pred = logprob.topk(k=1,dim=0)[1].view(-1)
for label in label2id:
if label == 'O':
continue
labelid = label2id[label]
if data.output.view(-1)[y_pred[labelid]] == labelid:
correct[label] += 1
else:
if output_file != None:
num_sent, sent_len, word_len = data.input_word.size(
)
id = y_pred[label2id[label]]
word = data.words[data.sents[int(
id / sent_len)][id % sent_len]]
output_file.write(
'%d %d %s %s\n' %
(data.set_id, data.fax_id, label, word))
return {label: float(correct[label]) / total for label in correct}
batch = 1
weight = torch.zeros(len(label2id))
for label, id in label2id.items():
weight[id] = 1 if label == 'O' else 10
loss_function = nn.NLLLoss(weight.cuda(), reduce=False)
optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad,
model.parameters()),
lr=args.lr / float(batch),
weight_decay=args.wd)
# scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=1, gamma=0.8)
best_acc = -1
wait = 0
for epoch in range(args.epochs):
sum_loss = 0
model.train()
# random.shuffle(dataset.train)
for idx, data in enumerate(dataset.train):
word, feat = Variable(data.input_word).cuda(), Variable(
data.input_feat).cuda()
a_ud, a_lr = Variable(data.a_ud,
requires_grad=False).cuda(), Variable(
data.a_lr,
requires_grad=False).cuda()
mask = Variable(data.mask, requires_grad=False).cuda()
true_output = Variable(data.output).cuda()
if args.globalnode:
logprob, form = model(word, feat, mask, a_ud, a_lr)
else:
logprob = model(word, feat, mask, a_ud, a_lr)
loss = torch.mean(
mask.view(-1) * loss_function(logprob.view(-1, output_dim),
true_output.view(-1)))
if args.globalnode:
true_form = Variable(torch.LongTensor([data.set_id - 1
])).cuda()
loss = loss + 0.1 * F.nll_loss(form, true_form)
sum_loss += loss.data.sum()
loss.backward()
if (idx + 1) % batch == 0 or idx + 1 == len(dataset.train):
optimizer.step()
optimizer.zero_grad()
train_acc = evaluate(model, dataset.train)
valid_acc = evaluate(model, dataset.valid)
test_acc = evaluate(model, dataset.test)
print('Epoch %d: Train Loss: %.3f Train: %s Valid: %s Test: %s' \
% (epoch, sum_loss, acc_to_str(train_acc), acc_to_str(valid_acc), acc_to_str(test_acc)))
# scheduler.step()
acc = np.log(list(valid_acc.values())).sum()
if epoch < 6:
continue
if acc >= best_acc:
torch.save(model.state_dict(), args.output + '.model')
wait = 0 if acc > best_acc else wait + 1
best_acc = max(acc, best_acc)
if wait >= args.patience:
break
model.load_state_dict(torch.load(args.output + '.model'))
test_acc = evaluate(model, dataset.test, output_file=output_file)
print('########', acc_to_str(test_acc))
for label in test_acc:
cross_res[label].append(test_acc[label])
print("Cross Validation Result:")
for label in cross_res:
cross_res[label] = np.mean(cross_res[label])
print(acc_to_str(cross_res))
return cross_res
def main():
# Training settings
parser = argparse.ArgumentParser(
description='GNN baselines on pcqm4m with PGL')
parser.add_argument('--use_cuda', action='store_true')
parser.add_argument('--device',
type=int,
default=0,
help='which gpu to use if any (default: 0)')
parser.add_argument(
'--gnn',
type=str,
default='gin-virtual',
help=
'GNN gin, gin-virtual, or gcn, or gcn-virtual (default: gin-virtual)')
parser.add_argument(
'--graph_pooling',
type=str,
default='sum',
help='graph pooling strategy mean or sum (default: sum)')
parser.add_argument('--drop_ratio',
type=float,
default=0,
help='dropout ratio (default: 0)')
parser.add_argument(
'--num_layers',
type=int,
default=5,
help='number of GNN message passing layers (default: 5)')
parser.add_argument(
'--emb_dim',
type=int,
default=600,
help='dimensionality of hidden units in GNNs (default: 600)')
parser.add_argument('--train_subset', action='store_true')
parser.add_argument('--batch_size',
type=int,
default=256,
help='input batch size for training (default: 256)')
parser.add_argument('--epochs',
type=int,
default=100,
help='number of epochs to train (default: 100)')
parser.add_argument('--num_workers',
type=int,
default=1,
help='number of workers (default: 1)')
parser.add_argument('--log_dir',
type=str,
default="",
help='tensorboard log directory')
parser.add_argument('--checkpoint_dir',
type=str,
default='',
help='directory to save checkpoint')
parser.add_argument('--save_test_dir',
type=str,
default='',
help='directory to save test submission file')
args = parser.parse_args()
print(args)
random.seed(42)
np.random.seed(42)
paddle.seed(42)
if not args.use_cuda:
paddle.set_device("cpu")
### automatic dataloading and splitting
class Config():
def __init__(self):
self.base_data_path = "./dataset"
config = Config()
ds = MolDataset(config)
split_idx = ds.get_idx_split()
test_ds = Subset(ds, split_idx['test'])
print("Test exapmles: ", len(test_ds))
### automatic evaluator. takes dataset name as input
evaluator = PCQM4MEvaluator()
test_loader = Dataloader(test_ds,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.num_workers,
collate_fn=CollateFn())
shared_params = {
'num_layers': args.num_layers,
'emb_dim': args.emb_dim,
'drop_ratio': args.drop_ratio,
'graph_pooling': args.graph_pooling
}
if args.gnn == 'gin':
model = GNN(gnn_type='gin', virtual_node=False, **shared_params)
elif args.gnn == 'gin-virtual':
model = GNN(gnn_type='gin', virtual_node=True, **shared_params)
elif args.gnn == 'gcn':
model = GNN(gnn_type='gcn', virtual_node=False, **shared_params)
elif args.gnn == 'gcn-virtual':
model = GNN(gnn_type='gcn', virtual_node=True, **shared_params)
else:
raise ValueError('Invalid GNN type')
num_params = sum(p.numel() for p in model.parameters())
print(f'#Params: {num_params}')
checkpoint_path = os.path.join(args.checkpoint_dir, 'checkpoint.pdparams')
if not os.path.exists(checkpoint_path):
raise RuntimeError(f'Checkpoint file not found at {checkpoint_path}')
model.set_state_dict(paddle.load(checkpoint_path))
print('Predicting on test data...')
y_pred = test(model, test_loader)
print('Saving test submission file...')
evaluator.save_test_submission({'y_pred': y_pred}, args.save_test_dir)
def main():
# Training settings
parser = argparse.ArgumentParser(description='GNN baselines on ogbgmol* data with Pytorch Geometrics')
parser.add_argument('--device', type=int, default=0,
help='which gpu to use if any (default: 0)')
parser.add_argument('--gnn', type=str, default='gin-virtual',
help='GNN gin, gin-virtual, or gcn, or gcn-virtual (default: gin-virtual)')
parser.add_argument('--drop_ratio', type=float, default=0.5,
help='dropout ratio (default: 0.5)')
parser.add_argument('--num_layer', type=int, default=5,
help='number of GNN message passing layers (default: 5)')
parser.add_argument('--pooling', type=str, default='mean',
help='Pooling tecnhnique for graph embedding')
parser.add_argument('--laf', type=str, default='mean',
help='Init function if laf pooling is specified')
parser.add_argument('--laf_layers', type=str, default='false',
help='If set to true, internal layers will be initialized with laf function')
parser.add_argument('--emb_dim', type=int, default=300,
help='dimensionality of hidden units in GNNs (default: 300)')
parser.add_argument('--batch_size', type=int, default=32,
help='input batch size for training (default: 32)')
parser.add_argument('--epochs', type=int, default=100,
help='number of epochs to train (default: 100)')
parser.add_argument('--num_workers', type=int, default=0,
help='number of workers (default: 0)')
parser.add_argument('--dataset', type=str, default="ogbg-molhiv",
help='dataset name (default: ogbg-molhiv)')
parser.add_argument('--feature', type=str, default="full",
help='full feature or simple feature')
parser.add_argument('--filename', type=str, default="",
help='filename to output result (default: )')
parser.add_argument('--seed', type=int, default=92,
help='torch seed')
parser.add_argument('--alternate', type=str, default='false',
help='use alternate learning with laf')
args = parser.parse_args()
print(args)
torch.manual_seed(args.seed)
device = torch.device("cuda:" + str(args.device)) if torch.cuda.is_available() else torch.device("cpu")
### automatic dataloading and splitting
dataset = PygGraphPropPredDataset(name=args.dataset)
if args.feature == 'full':
pass
elif args.feature == 'simple':
print('using simple feature')
# only retain the top two node/edge features
dataset.data.x = dataset.data.x[:, :2]
dataset.data.edge_attr = dataset.data.edge_attr[:, :2]
split_idx = dataset.get_idx_split()
### automatic evaluator. takes dataset name as input
evaluator = Evaluator(args.dataset)
train_loader = DataLoader(dataset[split_idx["train"]], batch_size=args.batch_size, shuffle=True,
num_workers=args.num_workers)
valid_loader = DataLoader(dataset[split_idx["valid"]], batch_size=args.batch_size, shuffle=False,
num_workers=args.num_workers)
test_loader = DataLoader(dataset[split_idx["test"]], batch_size=args.batch_size, shuffle=False,
num_workers=args.num_workers)
if args.gnn == 'gin':
model = GNN(gnn_type='gin', num_tasks=dataset.num_tasks, emb_dim=args.emb_dim, drop_ratio=args.drop_ratio,
virtual_node=False, graph_pooling=args.pooling, laf_fun=args.laf, laf_layers=args.laf_layers,
device=device, lafgrad=True).to(device)
elif args.gnn == 'gin-virtual':
model = GNN(gnn_type='gin', num_tasks=dataset.num_tasks, emb_dim=args.emb_dim, drop_ratio=args.drop_ratio,
virtual_node=True, graph_pooling=args.pooling, laf_fun=args.laf, laf_layers=args.laf_layers,
device=device, lafgrad=True).to(device)
elif args.gnn == 'gcn':
model = GNN(gnn_type='gcn', num_tasks=dataset.num_tasks, emb_dim=args.emb_dim, drop_ratio=args.drop_ratio,
virtual_node=False, graph_pooling=args.pooling, laf_fun=args.laf, laf_layers=args.laf_layers,
device=device, lafgrad=True).to(device)
elif args.gnn == 'gcn-virtual':
model = GNN(gnn_type='gcn', num_tasks=dataset.num_tasks, emb_dim=args.emb_dim, drop_ratio=args.drop_ratio,
virtual_node=True, graph_pooling=args.pooling, laf_fun=args.laf, laf_layers=args.laf_layers,
device=device, lafgrad=True).to(device)
elif args.gnn == 'gat':
model = GNN(gnn_type='gat', num_tasks=dataset.num_tasks, emb_dim=args.emb_dim, drop_ratio=args.drop_ratio,
virtual_node=False, graph_pooling=args.pooling, laf_fun=args.laf, laf_layers=args.laf_layers,
device=device, lafgrad=True).to(device)
else:
raise ValueError('Invalid GNN type')
#model.load_state_dict(torch.load("{}_fixed_training.mdl".format(args.filename)))
model_params = []
laf_params = []
for n, p in model.named_parameters():
if n == 'pool.weights' or n == 'pool.alpha' or n == 'pool.beta' or n == 'pool.N' or n == 'pool.M':
laf_params.append(p)
else:
model_params.append(p)
optimizer = optim.Adam(model_params, lr=0.001)
if laf_params == []:
optimizerlaf = None
else:
optimizerlaf = optim.Adam(laf_params, lr=0.0001)
flog = open(args.filename + ".log", 'a')
valid_curve = []
test_curve = []
train_curve = []
if 'classification' in dataset.task_type:
best_val = 0
else:
best_val = 1e12
flog.write("{}\n".format(args))
bflag = True
for epoch in range(1, args.epochs + 1):
start = time.time()
print("=====Epoch {}".format(epoch))
flog.write("=====Epoch {}\n".format(epoch))
print('Training...')
#if args.alternate == 'false':
# train_perf = train(model, device, train_loader, optimizer, optimizerlaf, dataset.task_type, evaluator)
#else:
# train_perf = train(model, device, train_loader, optimizer, None, dataset.task_type, evaluator)
#if args.alternate == 'false':
train_perf = train(model, device, train_loader, optimizer, optimizerlaf, dataset.task_type, evaluator, alternate=args.alternate)
print('Evaluating...')
# train_perf = eval(model, device, train_loader, evaluator)
valid_perf = eval(model, device, valid_loader, evaluator)
test_perf = eval(model, device, test_loader, evaluator)
print({'Train': train_perf, 'Validation': valid_perf, 'Test': test_perf})
print("Time {:.4f}s".format(time.time() - start))
if laf_params != []:
print("{}\n".format(torch.norm(model.pool.weights)))
flog.write("{}\n".format({'Train': train_perf, 'Validation': valid_perf, 'Test': test_perf}))
flog.write("Time: {}\n".format(time.time()-start))
if laf_params != []:
flog.write("Laf weights norm: {}\n".format(torch.norm(model.pool.weights, dim=0)))
flog.flush()
train_curve.append(train_perf[dataset.eval_metric])
valid_curve.append(valid_perf[dataset.eval_metric])
test_curve.append(test_perf[dataset.eval_metric])
if 'classification' in dataset.task_type:
if valid_perf[dataset.eval_metric] >= best_val:
best_val = valid_perf[dataset.eval_metric]
if not args.filename == '':
if args.alternate == 'true':
torch.save(model.state_dict(), '{}_fixed_training.mdl'.format(args.filename))
else:
torch.save(model.state_dict(), '{}.mdl'.format(args.filename))
else:
if valid_perf[dataset.eval_metric] <= best_val:
best_val = epoch
if not args.filename == '':
if args.alternate == 'true':
torch.save(model.state_dict(), '{}_fixed_training.mdl'.format(args.filename))
else:
torch.save(model.state_dict(), '{}.mdl'.format(args.filename))
if 'classification' in dataset.task_type:
best_val_epoch = np.argmax(np.array(valid_curve))
best_train = max(train_curve)
else:
best_val_epoch = np.argmin(np.array(valid_curve))
best_train = min(train_curve)
print('Finished training!')
print('Best validation score: {}'.format(valid_curve[best_val_epoch]))
print('Test score: {}'.format(test_curve[best_val_epoch]))
flog.write('Finished training!\n')
flog.write('Best validation score: {}\n'.format(valid_curve[best_val_epoch]))
flog.write('Test score: {}\n'.format(test_curve[best_val_epoch]))
flog.flush()
if not args.filename == '':
torch.save({'Val': valid_curve[best_val_epoch], 'Test': test_curve[best_val_epoch],
'Train': train_curve[best_val_epoch], 'BestTrain': best_train}, args.filename + "_fixed_training.res")
# if args.alternate == 'true'and optimizerlaf:
# args.alternate = 'false'
# flog.write("===================LAF TRAINING=================\n")
# valid_curve = []
# test_curve = []
# train_curve = []
# if 'classification' in dataset.task_type:
# best_val = 0
# else:
# best_val = 1e12
# for epoch in range(1, args.epochs + 1):
# start = time.time()
# print("=====Epoch {}".format(epoch))
# flog.write("=====Epoch {}\n".format(epoch))
# print('Training...')
# train_perf = train(model, device, train_loader, optimizerlaf, None, dataset.task_type, evaluator)
# print('Evaluating...')
# # train_perf = eval(model, device, train_loader, evaluator)
# valid_perf = eval(model, device, valid_loader, evaluator)
# test_perf = eval(model, device, test_loader, evaluator)
# print({'Train': train_perf, 'Validation': valid_perf, 'Test': test_perf})
# print("Time {:.4f}s".format(time.time() - start))
# #print("{}\n".format(torch.norm(model.pool.weights)))
# flog.write("{}\n".format({'Train': train_perf, 'Validation': valid_perf, 'Test': test_perf}))
# flog.write("Time: {}\n".format(time.time()-start))
# #flog.write("Laf weights norm: {}\n".format(torch.norm(model.pool.weights, dim=0)))
# flog.flush()
# train_curve.append(train_perf[dataset.eval_metric])
# valid_curve.append(valid_perf[dataset.eval_metric])
# test_curve.append(test_perf[dataset.eval_metric])
# if 'classification' in dataset.task_type:
# if valid_perf[dataset.eval_metric] >= best_val:
# best_val = valid_perf[dataset.eval_metric]
# if not args.filename == '':
# torch.save(model.state_dict(), '{}_laf_training.mdl'.format(args.filename))
# else:
# if valid_perf[dataset.eval_metric] <= best_val:
# best_val = epoch
# if not args.filename == '':
# torch.save(model.state_dict(), '{}_laf_training.mdl'.format(args.filename))
# if 'classification' in dataset.task_type:
# best_val_epoch = np.argmax(np.array(valid_curve))
# best_train = max(train_curve)
# else:
# best_val_epoch = np.argmin(np.array(valid_curve))
# best_train = min(train_curve)
# print('Finished training!')
# print('Best validation score: {}'.format(valid_curve[best_val_epoch]))
# print('Test score: {}'.format(test_curve[best_val_epoch]))
# flog.write('Finished training!\n')
# flog.write('Best validation score: {}\n'.format(valid_curve[best_val_epoch]))
# flog.write('Test score: {}\n'.format(test_curve[best_val_epoch]))
# flog.flush()
# if not args.filename == '':
# torch.save({'Val': valid_curve[best_val_epoch], 'Test': test_curve[best_val_epoch],
# 'Train': train_curve[best_val_epoch], 'BestTrain': best_train}, args.filename + "_laf_training.res")
flog.close()
def main():
# Training settings
parser = argparse.ArgumentParser(
description='GNN baselines on pcqm4m with Pytorch Geometrics')
parser.add_argument('--device',
type=int,
default=0,
help='which gpu to use if any (default: 0)')
parser.add_argument(
'--gnn',
type=str,
default='gin-virtual',
help=
'GNN gin, gin-virtual, or gcn, or gcn-virtual (default: gin-virtual)')
parser.add_argument(
'--graph_pooling',
type=str,
default='sum',
help='graph pooling strategy mean or sum (default: sum)')
parser.add_argument('--drop_ratio',
type=float,
default=0,
help='dropout ratio (default: 0)')
parser.add_argument(
'--num_layers',
type=int,
default=5,
help='number of GNN message passing layers (default: 5)')
parser.add_argument(
'--emb_dim',
type=int,
default=600,
help='dimensionality of hidden units in GNNs (default: 600)')
parser.add_argument('--train_subset', action='store_true')
parser.add_argument('--batch_size',
type=int,
default=256,
help='input batch size for training (default: 256)')
parser.add_argument('--epochs',
type=int,
default=100,
help='number of epochs to train (default: 100)')
parser.add_argument('--num_workers',
type=int,
default=0,
help='number of workers (default: 0)')
parser.add_argument('--log_dir',
type=str,
default="",
help='tensorboard log directory')
parser.add_argument('--checkpoint_dir',
type=str,
default='',
help='directory to save checkpoint')
parser.add_argument('--save_test_dir',
type=str,
default='',
help='directory to save test submission file')
args = parser.parse_args()
print(args)
np.random.seed(42)
torch.manual_seed(42)
torch.cuda.manual_seed(42)
random.seed(42)
device = torch.device(
"cuda:" +
str(args.device)) if torch.cuda.is_available() else torch.device("cpu")
### automatic dataloading and splitting
dataset = PygPCQM4MDataset(root='dataset/')
split_idx = dataset.get_idx_split()
### automatic evaluator. takes dataset name as input
evaluator = PCQM4MEvaluator()
if args.train_subset:
subset_ratio = 0.1
subset_idx = torch.randperm(len(
split_idx["train"]))[:int(subset_ratio * len(split_idx["train"]))]
train_loader = DataLoader(dataset[split_idx["train"][subset_idx]],
batch_size=args.batch_size,
shuffle=True,
num_workers=args.num_workers)
else:
train_loader = DataLoader(dataset[split_idx["train"]],
batch_size=args.batch_size,
shuffle=True,
num_workers=args.num_workers)
valid_loader = DataLoader(dataset[split_idx["valid"]],
batch_size=args.batch_size,
shuffle=False,
num_workers=args.num_workers)
if args.save_test_dir != '':
test_loader = DataLoader(dataset[split_idx["test-dev"]],
batch_size=args.batch_size,
shuffle=False,
num_workers=args.num_workers)
if args.checkpoint_dir != '':
os.makedirs(args.checkpoint_dir, exist_ok=True)
shared_params = {
'num_layers': args.num_layers,
'emb_dim': args.emb_dim,
'drop_ratio': args.drop_ratio,
'graph_pooling': args.graph_pooling
}
if args.gnn == 'gin':
model = GNN(gnn_type='gin', virtual_node=False,
**shared_params).to(device)
elif args.gnn == 'gin-virtual':
model = GNN(gnn_type='gin', virtual_node=True,
**shared_params).to(device)
elif args.gnn == 'gcn':
model = GNN(gnn_type='gcn', virtual_node=False,
**shared_params).to(device)
elif args.gnn == 'gcn-virtual':
model = GNN(gnn_type='gcn', virtual_node=True,
**shared_params).to(device)
else:
raise ValueError('Invalid GNN type')
num_params = sum(p.numel() for p in model.parameters())
print(f'#Params: {num_params}')
optimizer = optim.Adam(model.parameters(), lr=0.001)
if args.log_dir != '':
writer = SummaryWriter(log_dir=args.log_dir)
best_valid_mae = 1000
if args.train_subset:
scheduler = StepLR(optimizer, step_size=300, gamma=0.25)
args.epochs = 1000
else:
scheduler = StepLR(optimizer, step_size=30, gamma=0.25)
for epoch in range(1, args.epochs + 1):
print("=====Epoch {}".format(epoch))
print('Training...')
train_mae = train(model, device, train_loader, optimizer)
print('Evaluating...')
valid_mae = eval(model, device, valid_loader, evaluator)
print({'Train': train_mae, 'Validation': valid_mae})
if args.log_dir != '':
writer.add_scalar('valid/mae', valid_mae, epoch)
writer.add_scalar('train/mae', train_mae, epoch)
if valid_mae < best_valid_mae:
best_valid_mae = valid_mae
if args.checkpoint_dir != '':
print('Saving checkpoint...')
checkpoint = {
'epoch': epoch,
'model_state_dict': model.state_dict(),
'optimizer_state_dict': optimizer.state_dict(),
'scheduler_state_dict': scheduler.state_dict(),
'best_val_mae': best_valid_mae,
'num_params': num_params
}
torch.save(checkpoint,
os.path.join(args.checkpoint_dir, 'checkpoint.pt'))
if args.save_test_dir != '':
print('Predicting on test data...')
y_pred = test(model, device, test_loader)
print('Saving test submission file...')
evaluator.save_test_submission({'y_pred': y_pred},
args.save_test_dir,
mode='test-dev')
scheduler.step()
print(f'Best validation MAE so far: {best_valid_mae}')
if args.log_dir != '':
writer.close()
def main():
device = torch.device(
"cuda:" +
str(args.device)) if torch.cuda.is_available() else torch.device("cpu")
### automatic dataloading and splitting
dataset = PygGraphPropPredDataset(name=args.dataset, transform=add_zeros)
split_idx = dataset.get_idx_split()
### automatic evaluator. takes dataset name as input
evaluator = Evaluator(args.dataset)
train_loader = DataLoader(dataset[split_idx["train"]],
batch_size=args.batch_size,
shuffle=True,
num_workers=args.num_workers)
valid_loader = DataLoader(dataset[split_idx["valid"]],
batch_size=args.batch_size,
shuffle=False,
num_workers=args.num_workers)
test_loader = DataLoader(dataset[split_idx["test"]],
batch_size=args.batch_size,
shuffle=False,
num_workers=args.num_workers)
vals, tests = [], []
for run in range(args.runs):
best_val, final_test = 0, 0
if args.gnn == 'gin':
model = GNN(gnn_type='gin',
num_class=dataset.num_classes,
num_layer=args.num_layer,
emb_dim=args.emb_dim,
drop_ratio=args.drop_ratio,
virtual_node=False).to(device)
elif args.gnn == 'gin-virtual':
model = GNN(gnn_type='gin',
num_class=dataset.num_classes,
num_layer=args.num_layer,
emb_dim=args.emb_dim,
drop_ratio=args.drop_ratio,
virtual_node=True).to(device)
elif args.gnn == 'gcn':
model = GNN(gnn_type='gcn',
num_class=dataset.num_classes,
num_layer=args.num_layer,
emb_dim=args.emb_dim,
drop_ratio=args.drop_ratio,
virtual_node=False).to(device)
elif args.gnn == 'gcn-virtual':
model = GNN(gnn_type='gcn',
num_class=dataset.num_classes,
num_layer=args.num_layer,
emb_dim=args.emb_dim,
drop_ratio=args.drop_ratio,
virtual_node=True).to(device)
else:
raise ValueError('Invalid GNN type')
optimizer = torch.optim.Adam(model.parameters(), lr=args.lr)
for epoch in range(1, args.epochs + 1):
loss = train(model, device, train_loader, optimizer, args)
if epoch > args.epochs // 2 and epoch % args.test_freq == 0 or epoch == args.epochs:
#4min
train_perf = eval(model, device, train_loader, evaluator)
valid_perf = eval(model, device, valid_loader, evaluator)
test_perf = eval(model, device, test_loader, evaluator)
result = (train_perf[dataset.eval_metric],
valid_perf[dataset.eval_metric],
test_perf[dataset.eval_metric])
_, val, tst = result
if val > best_val:
best_val = val
final_test = tst
print(f'Run{run} val:{best_val}, test:{final_test}')
vals.append(best_val)
tests.append(final_test)
print('')
print(f"Average val accuracy: {np.mean(vals)} ± {np.std(vals)}")
print(f"Average test accuracy: {np.mean(tests)} ± {np.std(tests)}")
def main():
# Training settings
parser = argparse.ArgumentParser(
description='GNN baselines on ogbg-code2 data with Pytorch Geometrics')
parser.add_argument('--device',
type=int,
default=0,
help='which gpu to use if any (default: 0)')
parser.add_argument(
'--gnn',
type=str,
default='gcn-virtual',
help=
'GNN gin, gin-virtual, or gcn, or gcn-virtual (default: gcn-virtual)')
parser.add_argument('--drop_ratio',
type=float,
default=0,
help='dropout ratio (default: 0)')
parser.add_argument('--max_seq_len',
type=int,
default=5,
help='maximum sequence length to predict (default: 5)')
parser.add_argument(
'--num_vocab',
type=int,
default=5000,
help=
'the number of vocabulary used for sequence prediction (default: 5000)'
)
parser.add_argument(
'--num_layer',
type=int,
default=5,
help='number of GNN message passing layers (default: 5)')
parser.add_argument(
'--emb_dim',
type=int,
default=300,
help='dimensionality of hidden units in GNNs (default: 300)')
parser.add_argument('--batch_size',
type=int,
default=128,
help='input batch size for training (default: 128)')
parser.add_argument('--epochs',
type=int,
default=25,
help='number of epochs to train (default: 25)')
parser.add_argument('--random_split', action='store_true')
parser.add_argument('--num_workers',
type=int,
default=0,
help='number of workers (default: 0)')
parser.add_argument('--dataset',
type=str,
default="ogbg-code2",
help='dataset name (default: ogbg-code2)')
parser.add_argument('--filename',
type=str,
default="",
help='filename to output result (default: )')
args = parser.parse_args()
print(args)
device = torch.device(
"cuda:" +
str(args.device)) if torch.cuda.is_available() else torch.device("cpu")
### automatic dataloading and splitting
dataset = PygGraphPropPredDataset(name=args.dataset)
seq_len_list = np.array([len(seq) for seq in dataset.data.y])
print('Target seqence less or equal to {} is {}%.'.format(
args.max_seq_len,
np.sum(seq_len_list <= args.max_seq_len) / len(seq_len_list)))
split_idx = dataset.get_idx_split()
if args.random_split:
print('Using random split')
perm = torch.randperm(len(dataset))
num_train, num_valid, num_test = len(split_idx['train']), len(
split_idx['valid']), len(split_idx['test'])
split_idx['train'] = perm[:num_train]
split_idx['valid'] = perm[num_train:num_train + num_valid]
split_idx['test'] = perm[num_train + num_valid:]
assert (len(split_idx['train']) == num_train)
assert (len(split_idx['valid']) == num_valid)
assert (len(split_idx['test']) == num_test)
# print(split_idx['train'])
# print(split_idx['valid'])
# print(split_idx['test'])
# train_method_name = [' '.join(dataset.data.y[i]) for i in split_idx['train']]
# valid_method_name = [' '.join(dataset.data.y[i]) for i in split_idx['valid']]
# test_method_name = [' '.join(dataset.data.y[i]) for i in split_idx['test']]
# print('#train')
# print(len(train_method_name))
# print('#valid')
# print(len(valid_method_name))
# print('#test')
# print(len(test_method_name))
# train_method_name_set = set(train_method_name)
# valid_method_name_set = set(valid_method_name)
# test_method_name_set = set(test_method_name)
# # unique method name
# print('#unique train')
# print(len(train_method_name_set))
# print('#unique valid')
# print(len(valid_method_name_set))
# print('#unique test')
# print(len(test_method_name_set))
# # unique valid/test method name
# print('#valid unseen during training')
# print(len(valid_method_name_set - train_method_name_set))
# print('#test unseen during training')
# print(len(test_method_name_set - train_method_name_set))
### building vocabulary for sequence predition. Only use training data.
vocab2idx, idx2vocab = get_vocab_mapping(
[dataset.data.y[i] for i in split_idx['train']], args.num_vocab)
# test encoder and decoder
# for data in dataset:
# # PyG >= 1.5.0
# print(data.y)
#
# # PyG 1.4.3
# # print(data.y[0])
# data = encode_y_to_arr(data, vocab2idx, args.max_seq_len)
# print(data.y_arr[0])
# decoded_seq = decode_arr_to_seq(data.y_arr[0], idx2vocab)
# print(decoded_seq)
# print('')
## test augment_edge
# data = dataset[2]
# print(data)
# data_augmented = augment_edge(data)
# print(data_augmented)
### set the transform function
# augment_edge: add next-token edge as well as inverse edges. add edge attributes.
# encode_y_to_arr: add y_arr to PyG data object, indicating the array representation of a sequence.
dataset.transform = transforms.Compose([
augment_edge,
lambda data: encode_y_to_arr(data, vocab2idx, args.max_seq_len)
])
### automatic evaluator. takes dataset name as input
evaluator = Evaluator(args.dataset)
train_loader = DataLoader(dataset[split_idx["train"]],
batch_size=args.batch_size,
shuffle=True,
num_workers=args.num_workers)
valid_loader = DataLoader(dataset[split_idx["valid"]],
batch_size=args.batch_size,
shuffle=False,
num_workers=args.num_workers)
test_loader = DataLoader(dataset[split_idx["test"]],
batch_size=args.batch_size,
shuffle=False,
num_workers=args.num_workers)
nodetypes_mapping = pd.read_csv(
os.path.join(dataset.root, 'mapping', 'typeidx2type.csv.gz'))
nodeattributes_mapping = pd.read_csv(
os.path.join(dataset.root, 'mapping', 'attridx2attr.csv.gz'))
print(nodeattributes_mapping)
### Encoding node features into emb_dim vectors.
### The following three node features are used.
# 1. node type
# 2. node attribute
# 3. node depth
node_encoder = ASTNodeEncoder(args.emb_dim,
num_nodetypes=len(nodetypes_mapping['type']),
num_nodeattributes=len(
nodeattributes_mapping['attr']),
max_depth=20)
if args.gnn == 'gin':
model = GNN(num_vocab=len(vocab2idx),
max_seq_len=args.max_seq_len,
node_encoder=node_encoder,
num_layer=args.num_layer,
gnn_type='gin',
emb_dim=args.emb_dim,
drop_ratio=args.drop_ratio,
virtual_node=False).to(device)
elif args.gnn == 'gin-virtual':
model = GNN(num_vocab=len(vocab2idx),
max_seq_len=args.max_seq_len,
node_encoder=node_encoder,
num_layer=args.num_layer,
gnn_type='gin',
emb_dim=args.emb_dim,
drop_ratio=args.drop_ratio,
virtual_node=True).to(device)
elif args.gnn == 'gcn':
model = GNN(num_vocab=len(vocab2idx),
max_seq_len=args.max_seq_len,
node_encoder=node_encoder,
num_layer=args.num_layer,
gnn_type='gcn',
emb_dim=args.emb_dim,
drop_ratio=args.drop_ratio,
virtual_node=False).to(device)
elif args.gnn == 'gcn-virtual':
model = GNN(num_vocab=len(vocab2idx),
max_seq_len=args.max_seq_len,
node_encoder=node_encoder,
num_layer=args.num_layer,
gnn_type='gcn',
emb_dim=args.emb_dim,
drop_ratio=args.drop_ratio,
virtual_node=True).to(device)
else:
raise ValueError('Invalid GNN type')
optimizer = optim.Adam(model.parameters(), lr=0.001)
print(f'#Params: {sum(p.numel() for p in model.parameters())}')
valid_curve = []
test_curve = []
train_curve = []
for epoch in range(1, args.epochs + 1):
print("=====Epoch {}".format(epoch))
print('Training...')
train(model, device, train_loader, optimizer)
print('Evaluating...')
train_perf = eval(
model,
device,
train_loader,
evaluator,
arr_to_seq=lambda arr: decode_arr_to_seq(arr, idx2vocab))
valid_perf = eval(
model,
device,
valid_loader,
evaluator,
arr_to_seq=lambda arr: decode_arr_to_seq(arr, idx2vocab))
test_perf = eval(
model,
device,
test_loader,
evaluator,
arr_to_seq=lambda arr: decode_arr_to_seq(arr, idx2vocab))
print({
'Train': train_perf,
'Validation': valid_perf,
'Test': test_perf
})
train_curve.append(train_perf[dataset.eval_metric])
valid_curve.append(valid_perf[dataset.eval_metric])
test_curve.append(test_perf[dataset.eval_metric])
print('F1')
best_val_epoch = np.argmax(np.array(valid_curve))
best_train = max(train_curve)
print('Finished training!')
print('Best validation score: {}'.format(valid_curve[best_val_epoch]))
print('Test score: {}'.format(test_curve[best_val_epoch]))
if not args.filename == '':
result_dict = {
'Val': valid_curve[best_val_epoch],
'Test': test_curve[best_val_epoch],
'Train': train_curve[best_val_epoch],
'BestTrain': best_train
}
torch.save(result_dict, args.filename)
def main(opt):
device = torch.device('cuda' if opt['cuda'] == True and torch.cuda.is_available() else 'cpu')
#--------------------------------------------------
# Load data.
#--------------------------------------------------
net_file = opt['dataset'] + '/net.txt'
label_file = opt['dataset'] + '/label.txt'
feature_file = opt['dataset'] + '/feature.txt'
train_file = opt['dataset'] + '/train.txt'
dev_file = opt['dataset'] + '/dev.txt'
test_file = opt['dataset'] + '/test.txt'
vocab_node = loader.Vocab(net_file, [0, 1])
vocab_label = loader.Vocab(label_file, [1])
vocab_feature = loader.Vocab(feature_file, [1])
opt['num_node'] = len(vocab_node)
opt['num_feature'] = len(vocab_feature)
opt['num_class'] = len(vocab_label)
graph = loader.Graph(file_name=net_file, entity=[vocab_node, 0, 1])
label = loader.EntityLabel(file_name=label_file, entity=[vocab_node, 0], label=[vocab_label, 1])
feature = loader.EntityFeature(file_name=feature_file, entity=[vocab_node, 0], feature=[vocab_feature, 1])
d = graph.to_symmetric(opt['self_link_weight'])
feature.to_one_hot(binary=True)
adj = graph.get_sparse_adjacency(opt['cuda'])
deg = torch.zeros(adj.shape[0])
for k,v in d.items():
deg[k] = v
with open(train_file, 'r') as fi:
idx_train = [vocab_node.stoi[line.strip()] for line in fi]
with open(dev_file, 'r') as fi:
idx_dev = [vocab_node.stoi[line.strip()] for line in fi]
with open(test_file, 'r') as fi:
idx_test = [vocab_node.stoi[line.strip()] for line in fi]
inputs = torch.Tensor(feature.one_hot)
target = torch.LongTensor(label.itol)
idx_train = torch.LongTensor(idx_train)
idx_dev = torch.LongTensor(idx_dev)
idx_test = torch.LongTensor(idx_test)
if opt['cuda']:
inputs = inputs.cuda()
target = target.cuda()
idx_train = idx_train.cuda()
idx_dev = idx_dev.cuda()
idx_test = idx_test.cuda()
#--------------------------------------------------
# Build model.
#--------------------------------------------------
if opt['weight']:
gnn = WGNN(opt, adj, deg, opt['time'])
else:
gnn = GNN(opt, adj, deg, opt['time'])
trainer = Trainer(opt, gnn)
print(gnn)
print(opt)
#--------------------------------------------------
# Train model.
#--------------------------------------------------
def train(epochs):
best = 0.0
results = []
prev_dev_acc = 0
cnt = 0
lr = opt['lr']
for epoch in range(0, epochs):
# -----------------------
# Train Model
# -----------------------
if opt['weight']:
loss = trainer.updatew(inputs, target, idx_train)
else:
loss = trainer.update(inputs, target, idx_train)
# -----------------------
# Evaluate Model
# -----------------------
_, preds, accuracy_dev = trainer.evaluate(inputs, target, idx_dev)
# -----------------------
# Test Model
# -----------------------
_, preds, accuracy_test = trainer.evaluate(inputs, target, idx_test)
print(
'Epoch: {} | Loss: {:.3f} | Dev acc: {:.3f} | Test acc: {:.3f} | Forward: {} {:.3f} | Backward: {} {:.3f}'.format(
epoch,
loss,
accuracy_dev,
accuracy_test,
trainer.fm.get_value(),
trainer.fm.get_average(),
trainer.bm.get_value(),
trainer.bm.get_average()))
results += [(accuracy_dev, accuracy_test)]
if accuracy_dev >= best:
best = accuracy_dev
state = dict([('model', copy.deepcopy(trainer.model.state_dict())),
('optim', copy.deepcopy(trainer.optimizer.state_dict()))])
trainer.model.load_state_dict(state['model'])
trainer.optimizer.load_state_dict(state['optim'])
return results
results = train(opt['epoch'])
def get_accuracy(results):
best_dev, acc_test = 0.0, 0.0
for d, t in results:
if d > best_dev:
best_dev, acc_test = d, t
return acc_test, best_dev
acc_test = get_accuracy(results)
print('{:.3f}'.format(acc_test[0]*100))
return acc_test
# retrieve input, arcnode, nodegraph and target for validation set
inp_val = val[fold][0]
arcnode_val = val[fold][1]
labels_val = val[fold][4]
nodegraph_val = val[fold][2]
# initialize network
net = n.Net(input_dim, state_dim, output_dim)
# instantiate GNN
g = GNN.GNN(net,
input_dim,
output_dim,
state_dim,
max_it,
optimizer,
learning_rate,
threshold,
graph_based=True,
param=param,
config=config)
# train GNN, and validate every 2 epochs, (early stopping)
count = 0
valid_best = None
patience = 0
for j in range(0, num_epoch):
g.Train(inp[0], arcnode[0], labels, count, nodegraph[0])
print("Epoch ", j)
if count % 2 == 0:
def main():
# Training settings
parser = argparse.ArgumentParser(
description='GNN baselines on ogbg-ppi data with Pytorch Geometrics')
parser.add_argument('--device',
type=int,
default=0,
help='which gpu to use if any (default: 0)')
parser.add_argument(
'--gnn',
type=str,
default='gin-virtual',
help=
'GNN gin, gin-virtual, or gcn, or gcn-virtual (default: gin-virtual)')
parser.add_argument('--drop_ratio',
type=float,
default=0.5,
help='dropout ratio (default: 0.5)')
parser.add_argument(
'--num_layer',
type=int,
default=5,
help='number of GNN message passing layers (default: 5)')
parser.add_argument(
'--emb_dim',
type=int,
default=300,
help='dimensionality of hidden units in GNNs (default: 300)')
parser.add_argument('--batch_size',
type=int,
default=32,
help='input batch size for training (default: 32)')
parser.add_argument('--epochs',
type=int,
default=100,
help='number of epochs to train (default: 100)')
parser.add_argument('--num_workers',
type=int,
default=0,
help='number of workers (default: 0)')
parser.add_argument('--dataset',
type=str,
default="ogbg-ppi",
help='dataset name (default: ogbg-ppi)')
parser.add_argument('--filename',
type=str,
default="",
help='filename to output result (default: )')
args = parser.parse_args()
device = torch.device(
"cuda:" +
str(args.device)) if torch.cuda.is_available() else torch.device("cpu")
### automatic dataloading and splitting
dataset = PygGraphPropPredDataset(name=args.dataset, transform=add_zeros)
splitted_idx = dataset.get_idx_split()
### automatic evaluator. takes dataset name as input
evaluator = Evaluator(args.dataset)
train_loader = DataLoader(dataset[splitted_idx["train"]],
batch_size=args.batch_size,
shuffle=True,
num_workers=args.num_workers)
valid_loader = DataLoader(dataset[splitted_idx["valid"]],
batch_size=args.batch_size,
shuffle=False,
num_workers=args.num_workers)
test_loader = DataLoader(dataset[splitted_idx["test"]],
batch_size=args.batch_size,
shuffle=False,
num_workers=args.num_workers)
if args.gnn == 'gin':
model = GNN(gnn_type='gin',
num_class=37,
emb_dim=args.emb_dim,
drop_ratio=args.drop_ratio,
virtual_node=False).to(device)
elif args.gnn == 'gin-virtual':
model = GNN(gnn_type='gin',
num_class=37,
emb_dim=args.emb_dim,
drop_ratio=args.drop_ratio,
virtual_node=True).to(device)
elif args.gnn == 'gcn':
model = GNN(gnn_type='gcn',
num_class=37,
emb_dim=args.emb_dim,
drop_ratio=args.drop_ratio,
virtual_node=False).to(device)
elif args.gnn == 'gcn-virtual':
model = GNN(gnn_type='gcn',
num_class=37,
emb_dim=args.emb_dim,
drop_ratio=args.drop_ratio,
virtual_node=True).to(device)
else:
raise ValueError('Invalid GNN type')
optimizer = optim.Adam(model.parameters(), lr=0.001)
valid_curve = []
test_curve = []
train_curve = []
for epoch in range(1, args.epochs + 1):
print("=====Epoch {}".format(epoch))
print('Training...')
train(model, device, train_loader, optimizer)
print('Evaluating...')
train_perf = eval(model, device, train_loader, evaluator)
valid_perf = eval(model, device, valid_loader, evaluator)
test_perf = eval(model, device, test_loader, evaluator)
print({
'Train': train_perf,
'Validation': valid_perf,
'Test': test_perf
})
train_curve.append(train_perf['acc'])
valid_curve.append(valid_perf['acc'])
test_curve.append(test_perf['acc'])
best_val_epoch = np.argmax(np.array(valid_curve))
best_train = max(train_curve)
print('Finished training!')
print('Best validation score: {}'.format(valid_curve[best_val_epoch]))
print('Test score: {}'.format(test_curve[best_val_epoch]))
if not args.filename == '':
torch.save(
{
'Val': valid_curve[best_val_epoch],
'Test': test_curve[best_val_epoch],
'Train': train_curve[best_val_epoch],
'BestTrain': best_train
}, args.filename)
def main():
# Training settings
parser = argparse.ArgumentParser(
description='GNN baselines on ogbg-ppa data with Pytorch Geometrics')
parser.add_argument('--device',
type=int,
default=0,
help='which gpu to use if any (default: 0)')
parser.add_argument(
'--gnn',
type=str,
default='gin-virtual',
help=
'GNN gin, gin-virtual, or gcn, or gcn-virtual (default: gin-virtual)')
parser.add_argument('--drop_ratio',
type=float,
default=0.5,
help='dropout ratio (default: 0.5)')
parser.add_argument(
'--num_layer',
type=int,
default=5,
help='number of GNN message passing layers (default: 5)')
parser.add_argument('--pooling',
type=str,
default='mean',
help='Pooling tecnhnique for graph embedding')
parser.add_argument('--laf',
type=str,
default='mean',
help='Init function if laf pooling is specified')
parser.add_argument(
'--laf_layers',
type=str,
default='false',
help=
'If set to true, internal layers will be initialized with laf function'
)
parser.add_argument(
'--emb_dim',
type=int,
default=300,
help='dimensionality of hidden units in GNNs (default: 300)')
parser.add_argument('--batch_size',
type=int,
default=32,
help='input batch size for training (default: 32)')
parser.add_argument('--epochs',
type=int,
default=100,
help='number of epochs to train (default: 100)')
parser.add_argument('--num_workers',
type=int,
default=0,
help='number of workers (default: 0)')
parser.add_argument('--dataset',
type=str,
default="ogbg-ppa",
help='dataset name (default: ogbg-ppa)')
parser.add_argument('--filename',
type=str,
default="",
help='filename to output result (default: )')
parser.add_argument('--seed', type=int, default=92, help='torch seed')
args = parser.parse_args()
print(args)
device = torch.device(
"cuda:" +
str(args.device)) if torch.cuda.is_available() else torch.device("cpu")
### automatic dataloading and splitting
dataset = PygGraphPropPredDataset(name=args.dataset, transform=add_zeros)
split_idx = dataset.get_idx_split()
### automatic evaluator. takes dataset name as input
evaluator = Evaluator(args.dataset)
train_loader = DataLoader(dataset[split_idx["train"]],
batch_size=args.batch_size,
shuffle=True,
num_workers=args.num_workers)
valid_loader = DataLoader(dataset[split_idx["valid"]],
batch_size=args.batch_size,
shuffle=False,
num_workers=args.num_workers)
test_loader = DataLoader(dataset[split_idx["test"]],
batch_size=args.batch_size,
shuffle=False,
num_workers=args.num_workers)
if args.gnn == 'gin':
model = GNN(gnn_type='gin',
emb_dim=args.emb_dim,
drop_ratio=args.drop_ratio,
virtual_node=False,
graph_pooling=args.pooling,
laf_fun=args.laf,
device=args.device).to(device)
elif args.gnn == 'gin-virtual':
model = GNN(gnn_type='gin',
emb_dim=args.emb_dim,
drop_ratio=args.drop_ratio,
virtual_node=True,
graph_pooling=args.pooling,
laf_fun=args.laf,
device=args.device).to(device)
elif args.gnn == 'gcn':
model = GNN(gnn_type='gcn',
emb_dim=args.emb_dim,
drop_ratio=args.drop_ratio,
virtual_node=False,
graph_pooling=args.pooling,
laf_fun=args.laf,
device=args.device).to(device)
elif args.gnn == 'gcn-virtual':
model = GNN(gnn_type='gcn',
emb_dim=args.emb_dim,
drop_ratio=args.drop_ratio,
virtual_node=True,
graph_pooling=args.pooling,
laf_fun=args.laf,
device=args.device).to(device)
else:
raise ValueError('Invalid GNN type')
optimizer = optim.Adam(model.parameters(), lr=0.001)
valid_curve = []
test_curve = []
train_curve = []
best_val = 0
flog = open(args.filename + ".log", 'w')
flog.write("{}\n".format(args))
for epoch in range(1, args.epochs + 1):
start = time.time()
print("=====Epoch {}".format(epoch))
flog.write("=====Epoch {}\n".format(epoch))
print('Training...')
train(model, device, train_loader, optimizer)
print('Evaluating...')
train_perf = eval(model, device, train_loader, evaluator)
valid_perf = eval(model, device, valid_loader, evaluator)
test_perf = eval(model, device, test_loader, evaluator)
print({
'Train': train_perf,
'Validation': valid_perf,
'Test': test_perf
})
print("Time {:.4f}s".format(time.time() - start))
flog.write("{}\tTime: {}s\n".format(
{
'Train': train_perf,
'Validation': valid_perf,
'Test': test_perf
},
time.time() - start))
flog.flush()
train_curve.append(train_perf['acc'])
valid_curve.append(valid_perf['acc'])
test_curve.append(test_perf['acc'])
if valid_perf[dataset.eval_metric] >= best_val:
best_val = valid_perf[dataset.eval_metric]
if not args.filename == '':
torch.save(model.state_dict(), '{}.mdl'.format(args.filename))
best_val_epoch = np.argmax(np.array(valid_curve))
best_train = max(train_curve)
print('Finished training!')
print('Best validation score: {}'.format(valid_curve[best_val_epoch]))
print('Test score: {}'.format(test_curve[best_val_epoch]))
flog.write('Finished training!\n')
flog.write('Best validation score: {}\n'.format(
valid_curve[best_val_epoch]))
flog.write('Test score: {}\n'.format(test_curve[best_val_epoch]))
flog.flush()
if not args.filename == '':
torch.save(
{
'Val': valid_curve[best_val_epoch],
'Test': test_curve[best_val_epoch],
'Train': train_curve[best_val_epoch],
'BestTrain': best_train
}, args.filename + ".res")
