def train_gcn(args):
exp_init(args.seed, gpu_id=args.gpu)
# ! config
cf = GraphSageConfig(args)
cf.device = th.device("cuda:0" if args.gpu >= 0 else "cpu")
# ! Load Graph
g, features, n_feat, cf.n_class, labels, train_x, val_x, test_x = preprocess_data(cf.dataset, cf.train_percentage)
features = features.to(cf.device)
g = dgl.add_self_loop(g).to(cf.device)
supervision = SimpleObject({'train_x': train_x, 'val_x': val_x, 'test_x': test_x, 'labels': labels})
# ! Train Init
print(f'{cf}\nStart training..')
model = SAGE(n_feat, cf.n_hidden, cf.n_class, cf.n_layer, F.relu, cf.dropout, cf.aggregator)
model.to(cf.device)
print(model)
optimizer = th.optim.Adam(
model.parameters(), lr=cf.lr, weight_decay=cf.weight_decay)
if cf.early_stop > 0:
stopper = EarlyStopping(patience=cf.early_stop, path=cf.checkpoint_file)
else:
stopper = None
# ! Train
trainer = FullBatchTrainer(model=model, g=g, cf=cf, features=features,
sup=supervision, stopper=stopper, optimizer=optimizer,
loss_func=th.nn.CrossEntropyLoss())
trainer.run()
trainer.eval_and_save()
return cf
def forward(self, template, source, maxiter=10): template, source, template_mean, source_mean = data_utils.preprocess_data(template, source, self.p0_zero_mean, self.p1_zero_mean) result = self.iclk(template, source, maxiter) result = data_utils.postprocess_data(result, template, source, template_mean, source_mean, self.p0_zero_mean, self.p1_zero_mean) return result
def test(self, N, train_set, test_set):
# Generate "new" data
train_data, test_data = generate_pair_sets(train_set, test_set, 0, N)
_, _, _, x_test, y_test, _ = preprocess_data(train_data, test_data, 0,
N)
# Compute the model's prediction
test_output = self.forward(x_test)
pred_test = torch.argmax(test_output, dim=0)
# Compare the prediction with the real values
correct_test = pred_test == y_test
acc = np.mean(correct_test.cpu().numpy())
return acc
def main(args):
"""
:return:
"""
args.task_type = 'regression'
args.model_name = 'dnn'
warnings.filterwarnings('ignore')
torch.set_default_dtype(torch.float64)
filename = args.dataname + str(args.num) + '_' + args.model_name + '_' + str(args.run_number)
args.n_properties = properties_map[args.dataname]
args.standardise = False
with open('results/{}/{}/{}/{}.txt'.format(args.dataname, args.task_type, args.model_name, filename), 'a') as f:
# Write hyperparameters to file
write_args(f, args)
f.flush()
args.file = f
# These are matrices of dimension [N_train, n_descriptors + n_properties],
# [N_test, n_descriptors + n_properties] respectively. NaNs are imputed for missing
# values. We will use the ECFP4 fingerprint descriptor (1024 bits, radius = 3)
x = np.load(args.directory + args.dataname + '_x' + str(args.num) +
'_train_fingerprints.npy', allow_pickle=True)
x_test = np.load(args.directory + args.dataname + '_x' + str(args.num) +
'_test_fingerprints.npy', allow_pickle=True)
# Preprocess the data: standardising, applying PCA transforms and selecting relevant descriptors if desired.
# Also converts x and x_test from numpy matrices to torch tensors.
x, x_test, args.means, args.stds = preprocess_data(x, x_test, args.n_properties, args.pca_components, args.task_type)
args.in_dim = x.shape[1]
if args.model_name == 'baseline':
f.write('\n ... predictions from baseline model.')
# The baseline corresponds to mean imputation (with predictive uncertainty equal to
# the standard deviation of the training set)
r2_scores, mlls, rmses = baseline_metrics_calculator(x, args.n_properties,
means=args.means,
stds=args.stds)
write_metrics(r2_scores, mlls, rmses, file=f, set_type='train')
r2_scores, mlls, rmses = baseline_metrics_calculator(x_test, args.n_properties,
means=args.means,
stds=args.stds)
write_metrics(r2_scores, mlls, rmses, file=f, set_type='test')
else:
f.write('\n ... building {} model'.format(args.model_name))
# Load model
model = RegressionWrapper(network=ProbabilisticVanillaNN(in_dim=args.in_dim, out_dim=args.n_properties,
hidden_dims=args.hidden_dims, restrict_var=False),
batch_size=args.batch_size, lr=args.lr, file=args.file)
file_path = os.path.dirname(f.name) + '/models/{}_{}_{}_{}.dat'.format(args.dataname, args.model_name,
args.run_number, args.task_type)
# If model was saved after last training procedure, reload to continue training from where you left off
if os.path.isfile(file_path):
model.network.load_state_dict(torch.load(file_path))
f.write('\n ... training model. \n')
# Train model
model.train_model(x=x, epochs=args.epochs, epoch_print_freq=50, means=args.means,
stds=args.stds)
# Save model
torch.save(model.network.state_dict(), file_path)
# Calculate performance metrics on the train and test datasets
r2_scores, mlls, rmses = model.metrics_calculator(x, save=False)
write_metrics(r2_scores, mlls, rmses, file=f, set_type='train')
r2_scores, mlls, rmses = model.metrics_calculator(x_test, save=True)
write_metrics(r2_scores, mlls, rmses, file=f, set_type='test')
def train_GSR(args):
# ! Init Environment
exp_init(args.seed if hasattr(args, 'seed') else 0, args.gpu)
# ! Import packages
# Note that the assignment of GPU-ID must be specified before torch/dgl is imported.
import torch as th
import dgl
from utils.data_utils import preprocess_data
from utils.early_stopper import EarlyStopping
from models.GSR.GSR import GSR_pretrain, GSR_finetune, para_copy
from models.GSR.config import GSRConfig
from models.GSR.data_utils import get_pretrain_loader, get_structural_feature
from models.GSR.cl_utils import MemoryMoCo, moment_update, NCESoftmaxLoss
from models.GSR.trainer import FullBatchTrainer
from models.GSR.trainGSR import train_GSR
from models.GSR.PolyLRDecay import PolynomialLRDecay
# ! Config
cf = GSRConfig(args)
cf.device = th.device("cuda:0" if args.gpu >= 0 else "cpu")
# ! Load Graph
g, features, cf.n_feat, cf.n_class, labels, train_x, val_x, test_x = \
preprocess_data(cf.dataset, cf.train_percentage)
feat = {'F': features, 'S': get_structural_feature(g, cf)}
cf.feat_dim = {v: feat.shape[1] for v, feat in feat.items()}
supervision = SimpleObject({
'train_x': train_x,
'val_x': val_x,
'test_x': test_x,
'labels': labels
})
# ! Train Init
print(f'{cf}\nStart training..')
p_model = GSR_pretrain(g, cf).to(cf.device)
# print(p_model)
# ! Train Phase 1: Pretrain
if cf.p_epochs > 0:
# os.remove(cf.pretrain_model_ckpt) # Debug Only
if os.path.exists(cf.pretrain_model_ckpt):
p_model.load_state_dict(
th.load(cf.pretrain_model_ckpt, map_location=cf.device))
print(f'Pretrain embedding loaded from {cf.pretrain_model_ckpt}')
else:
print(
f'>>>> PHASE 1 - Pretraining and Refining Graph Structure <<<<<'
)
views = ['F', 'S']
optimizer = th.optim.Adam(p_model.parameters(),
lr=cf.prt_lr,
weight_decay=cf.weight_decay)
if cf.p_schedule_step > 1:
scheduler_poly_lr_decay = PolynomialLRDecay(
optimizer,
max_decay_steps=cf.p_schedule_step,
end_learning_rate=0.0001,
power=2.0)
# Construct virtual relation triples
p_model_ema = GSR_pretrain(g, cf).to(cf.device)
moment_update(p_model, p_model_ema, 0) # Copy
moco_memories = {
v: MemoryMoCo(
cf.n_hidden,
cf.nce_k, # Single-view contrast
cf.nce_t,
device=cf.device).to(cf.device)
for v in views
}
criterion = NCESoftmaxLoss(cf.device)
pretrain_loader = get_pretrain_loader(g.cpu(), cf)
for epoch_id in range(cf.p_epochs):
for step, (input_nodes, edge_subgraph,
blocks) in enumerate(pretrain_loader):
t0 = time()
blocks = [b.to(cf.device) for b in blocks]
edge_subgraph = edge_subgraph.to(cf.device)
input_feature = {
v: feat[v][input_nodes].to(cf.device)
for v in views
}
# ===================Moco forward=====================
p_model.train()
q_emb = p_model(edge_subgraph,
blocks,
input_feature,
mode='q')
std_dict = {
v: round(q_emb[v].std(dim=0).mean().item(), 4)
for v in ['F', 'S']
}
print(f"Std: {std_dict}")
if std_dict['F'] == 0 or std_dict['S'] == 0:
print(
f'\n\n????!!!! Same Embedding Epoch={epoch_id}Step={step}\n\n'
)
# q_emb = p_model(edge_subgraph, blocks, input_feature, mode='q')
with th.no_grad():
k_emb = p_model_ema(edge_subgraph,
blocks,
input_feature,
mode='k')
intra_out, inter_out = [], []
for tgt_view, memory in moco_memories.items():
for src_view in views:
if src_view == tgt_view:
intra_out.append(
memory(q_emb[f'{tgt_view}'],
k_emb[f'{tgt_view}']))
else:
inter_out.append(
memory(q_emb[f'{src_view}->{tgt_view}'],
k_emb[f'{tgt_view}']))
# ===================backward=====================
# ! Self-Supervised Learning
intra_loss = th.stack(
[criterion(out_) for out_ in intra_out]).mean()
inter_loss = th.stack(
[criterion(out_) for out_ in inter_out]).mean()
# ! Loss Fusion
loss_tensor = th.stack([intra_loss, inter_loss])
intra_w = float(cf.intra_weight)
loss_weights = th.tensor([intra_w, 1 - intra_w],
device=cf.device)
loss = th.dot(loss_weights, loss_tensor)
# ! Semi-Supervised Learning
optimizer.zero_grad()
loss.backward()
optimizer.step()
moment_update(p_model, p_model_ema, cf.momentum_factor)
print_log({
'Epoch': epoch_id,
'Batch': step,
'Time': time() - t0,
'intra_loss': intra_loss.item(),
'inter_loss': inter_loss.item(),
'overall_loss': loss.item()
})
if cf.p_schedule_step > 1:
scheduler_poly_lr_decay.step()
epochs_to_save = P_EPOCHS_SAVE_LIST + (
[1, 2, 3, 4] if args.dataset == 'arxiv' else [])
if epoch_id + 1 in epochs_to_save:
# Convert from p_epochs to current p_epoch checkpoint
ckpt_name = cf.pretrain_model_ckpt.replace(
f'_pi{cf.p_epochs}', f'_pi{epoch_id + 1}')
th.save(p_model.state_dict(), ckpt_name)
print(f'Model checkpoint {ckpt_name} saved.')
th.save(p_model.state_dict(), cf.pretrain_model_ckpt)
# ! Train Phase 2: Graph Structure Refine
print(f'>>>> PHASE 2 - Graph Structure Refine <<<<< ')
if cf.p_epochs <= 0 or cf.add_ratio + cf.rm_ratio == 0:
print('Use original graph!')
g_new = g
else:
if os.path.exists(cf.refined_graph_file):
print(f'Refined graph loaded from {cf.refined_graph_file}')
g_new = dgl.load_graphs(cf.refined_graph_file)[0][0]
else:
g_new = p_model.refine_graph(g, feat)
dgl.save_graphs(cf.refined_graph_file, [g_new])
# ! Train Phase 3: Node Classification
f_model = GSR_finetune(cf).to(cf.device)
print(f_model)
# Copy parameters
if cf.p_epochs > 0:
para_copy(f_model,
p_model.encoder.F,
paras_to_copy=['conv1.weight', 'conv1.bias'])
optimizer = th.optim.Adam(f_model.parameters(),
lr=cf.lr,
weight_decay=cf.weight_decay)
stopper = EarlyStopping(patience=cf.early_stop,
path=cf.checkpoint_file) if cf.early_stop else None
del g, feat, p_model
th.cuda.empty_cache()
print(f'>>>> PHASE 3 - Node Classification <<<<< ')
trainer_func = FullBatchTrainer
trainer = trainer_func(model=f_model,
g=g_new,
features=features,
sup=supervision,
cf=cf,
stopper=stopper,
optimizer=optimizer,
loss_func=th.nn.CrossEntropyLoss())
trainer.run()
trainer.eval_and_save()
return cf
def main(args):
"""
:return:
"""
probe_file = 'Probes_fingerprints.npy'
args.task_type = 'classification'
args.model_name = 'dnn'
warnings.filterwarnings('ignore')
torch.set_default_dtype(torch.float64)
filename = args.dataname + str(
args.num) + '_' + args.model_name + '_' + str(
args.run_number) + 'probes'
args.n_properties = properties_map[args.dataname]
args.standardise = False
with open(
'results/{}/{}/{}/{}.txt'.format(args.dataname, args.task_type,
args.model_name, filename),
'a') as f:
# Write hyperparameters to file
write_args(f, args)
f.flush()
args.file = f
# These are matrices of dimension [N_train, n_descriptors + n_properties],
# [N_test, n_descriptors + n_properties] respectively. NaNs are imputed for missing
# values. We will use the ECFP4 fingerprint descriptor (1024 bits, radius = 3)
x = np.load(args.directory + args.dataname + '_x' + str(args.num) +
'_train_fingerprints.npy',
allow_pickle=True)
x_test = np.load(args.directory + probe_file, allow_pickle=True)
y = np.empty((x_test.shape[0], args.n_properties))
x_test = np.concatenate((x_test, y), axis=1)
pdb.set_trace()
# Preprocess the data: standardising, applying PCA transforms and selecting relevant descriptors if desired.
# Also converts x and x_test from numpy matrices to torch tensors.
x, x_test, args.means, args.stds = preprocess_data(
x, x_test, args.n_properties, args.pca_components, args.task_type)
pdb.set_trace()
args.in_dim = x.shape[1]
f.write('\n ... building {} model'.format(args.model_name))
# Load model
model = ClassificationWrapper(network=BinaryClassificationNN(
in_dim=args.in_dim,
out_dim=args.n_properties,
hidden_dims=args.hidden_dims),
batch_size=args.batch_size,
lr=args.lr,
file=args.file)
file_path = os.path.dirname(
args.file.name) + '/models/{}_{}_{}_{}.dat'.format(
args.dataname, args.model_name, args.run_number,
args.task_type)
# If model was saved after last training procedure, reload parameters to continue training from where you left off
if os.path.isfile(file_path):
pdb.set_trace()
model.network.load_state_dict(torch.load(file_path))
else:
raise Exception('No model found.')
# Make predictions
predictions = model.predict(x_test, save=True, means=args.means)
pdb.set_trace()
"--gpu",
default=0,
type=int,
help="GPU id to use.")
parser.add_argument("-d", "--dataset", type=str, default=dataset)
parser.add_argument("-b",
"--block_log",
action="store_true",
help="block log or not")
parser.add_argument("-t", "--train_percentage", default=10, type=int)
parser.add_argument("--seed", default=0)
args = parser.parse_args()
train_percentage = 10
load_device = torch.device('cpu')
graph, features, n_feat, n_class, labels, train_x, val_x, test_x = preprocess_data(
dataset, train_percentage, load_device)
graph = graph_normalization(graph, False)
train_y, val_y, test_y = labels[train_x], labels[val_x], labels[test_x]
K = 2
new_graph = emb_generator(graph, features, K, args)
# emb = new_graph.ndata['se'].numpy()
# LR = LogisticRegression()
# LR.fit(emb[train_x.numpy()], train_y.numpy())
# val_pred = torch.tensor(LR.predict(emb[val_x.numpy()]))
# test_pred = torch.tensor(LR.predict(emb[test_x.numpy()]))
#
# val_acc = accuracy(val_pred, val_y)
# test_acc = accuracy(test_pred, test_y)
#
FONT_SIZE = 18
def h**o(graph, labels):
graph = dgl.remove_self_loop(graph)
src, dst = graph.edges()
intra_num = (labels[src] == labels[dst]).long().sum().numpy()
inter_num = (labels[src] != labels[dst]).long().sum().numpy()
return int(intra_num), int(inter_num)
dataset = 'arxiv'
train_percentage = 0
g_ori, features, n_feat, n_class, labels, train_x, val_x, test_x = preprocess_data(
dataset, train_percentage)
homo_dict = {}
homo_dict['intra'] = np.zeros((11, 11)).astype(int)
homo_dict['inter'] = np.zeros((11, 11)).astype(int)
homo_dict['homo_ratio'] = np.zeros((11, 11)).astype(float)
filebase = 'PR-_lr0.001_bsz1024_pi2_encGCN_dec-l2_hidden48-prt_intra_w-0.5_ncek16382_fanout5_10_prdo0_act_Elu_d256_GR-fsim_norm1_fsim_weight0.0_add0.0_rm0.25.bin'
filebase = filebase.split('fsim_weight')[0]
fsim_weight = 0.0
add_list = [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]
rm_list = [0.0, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5]
for aid in range(len(add_list)):
add = add_list[aid]
for rid in range(len(rm_list)):
def full_train_test(optimizer,
N_train,
N_test,
n_iter,
n_epochs,
batch_size=1,
d1=200,
d2=200,
d3=200,
gamma=1,
alpha=1,
rho=1,
verbose=False):
# Initialize metrics arrays
accuracy_test_array = []
accuracy_train_array = []
tr_acc_evolution = []
te_acc_evolution = []
time_array = []
# Generate the sets
train_set, test_set = get_sets()
# Generate train and test DataLoader
train_data, test_data = generate_pair_sets(train_set, test_set, N_train,
N_test, batch_size)
d0 = 1 * 28 * 28
for i in range(1, n_iter + 1):
print("Iteration %d" % i)
if (optimizer == "BCD"):
train_input, train_target, y_train_1hot, test_input, test_target, y_test_1hot = preprocess_data(
train_data, test_data, N_train, N_test)
# Instantiate the model
model = ModelBCD(d0, d1, d2, d3, 10, gamma, alpha, rho)
start_time = time.time()
# Train the model
tr_acc, te_acc = model.train(n_epochs,
train_input,
train_target,
y_train_1hot,
test_input,
test_target,
y_test_1hot,
verbose=verbose)
else:
# Instantiate the model
model = ModelDFW.ModelDFW(d0, d1, d2, d3, 10)
start_time = time.time()
# Train the model
tr_acc, te_acc = model.train(train_data,
test_data,
n_epochs,
verbose=verbose)
end_time = time.time()
# Store the train and test accuracy for each epoch of the iteration
tr_acc_evolution.append(tr_acc)
te_acc_evolution.append(te_acc)
# Computes test accuracy with "new" data
if (optimizer == "BCD"):
acc_test = model.test(N_test, train_set, test_set)
else:
_, test_data = generate_pair_sets(train_set, test_set, N_train,
N_test, batch_size)
acc_test = model.test(test_data, batch_size)
# Store the test accuracies for each iteration
accuracy_test_array.append(acc_test)
# Store the train accuracies for each iteration
accuracy_train_array.append(tr_acc[n_epochs - 1])
# Store the duration of the iteraton
time_array.append(end_time - start_time)
# Compute the mean and std of the test accuracies
acc_mean, acc_std = extract_mean_std(accuracy_test_array)
# Compute the mean and std of the duration an iteraion
time_mean, time_std = extract_mean_std(time_array)
print("Accuracy: %.3f +/- %.3f" % (acc_mean, acc_std))
print("Iteration time: %.3f +/- %.3f seconds" % (time_mean, time_std))
return tr_acc_evolution, te_acc_evolution, accuracy_test_array, time_array