def main():
# Load data
start = time.time()
N, _adj, _feats, _labels, train_adj, train_feats, train_nodes, val_nodes, test_nodes, y_train, y_val, y_test, val_mask, test_mask = utils.load_data(args.dataset)
print('Loaded data in {:.2f} seconds!'.format(time.time() - start))
# Prepare Train Data
start = time.time()
_, parts = utils.partition_graph(train_adj, train_nodes, args.num_clusters_train)
parts = [np.array(pt) for pt in parts]
train_features, train_support, y_train = utils.preprocess_multicluster(train_adj, parts, train_feats, y_train, args.num_clusters_train, args.batch_size)
print('Train Data pre-processed in {:.2f} seconds!'.format(time.time() - start))
# Prepare Test Data
if args.test == 1:
y_test, test_mask = y_val, val_mask
start = time.time()
_, test_features, test_support, y_test, test_mask = utils.preprocess(_adj, _feats, y_test, np.arange(N), args.num_clusters_test, test_mask)
print('Test Data pre-processed in {:.2f} seconds!'.format(time.time() - start))
# Shuffle Batches
batch_idxs = list(range(len(train_features)))
# model
model = GCN(fan_in=_in, fan_out=_out, layers=args.layers, dropout=args.dropout, normalize=True, bias=False).float()
model.cuda()
# Loss Function
criterion = torch.nn.CrossEntropyLoss()
# Optimization Algorithm
optimizer = torch.optim.Adam(model.parameters(), lr=args.lr)
# Learning Rate Schedule
scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, max_lr=args.lr, steps_per_epoch=int(args.num_clusters_train/args.batch_size), epochs=args.epochs+1, anneal_strategy='linear')
model.train()
# Train
for epoch in range(args.epochs + 1):
np.random.shuffle(batch_idxs)
avg_loss = 0
start = time.time()
for batch in batch_idxs:
loss = train(model.cuda(), criterion, optimizer, train_features[batch], train_support[batch], y_train[batch], dataset=args.dataset)
if args.lr_scheduler == 1:
scheduler.step()
avg_loss += loss.item()
# Write Train stats to tensorboard
writer.add_scalar('time/train', time.time() - start, epoch)
writer.add_scalar('loss/train', avg_loss/len(train_features), epoch)
if args.test == 1:
# Test on cpu
f1 = test(model.cpu(), test_features, test_support, y_test, test_mask, device='cpu')
print('f1: {:.4f}'.format(f1))
def test_memorize_minibatch(self):
for db_name in self.db_names:
db_info = get_db_info(db_name)
train_data, val_data, _ = get_train_val_test_datasets(
dataset_name=db_name,
train_test_split='use_full_train',
encoders=dict(CATEGORICAL='CategoricalOrdinalEnc',
SCALAR='ScalarRobustScalerEnc',
DATETIME='DatetimeScalarEnc',
LATLONG='LatLongScalarEnc',
TEXT='TextSummaryScalarEnc'),
)
train_loader = get_dataloader(
dataset=train_data,
batch_size=256,
sampler_class_name='SequentialSampler',
num_workers=0,
max_nodes_per_graph=False)
writer = DummyWriter()
model = GCN(writer,
db_info=db_info,
hidden_dim=256,
n_init_layers=3,
activation_class_name='SELU',
activation_class_kwargs={},
loss_class_kwargs={},
loss_class_name='CrossEntropyLoss',
p_dropout=0.0,
drop_whole_embeddings=True,
n_layers=3,
readout_class_name='AvgPooling',
readout_kwargs={})
if torch.cuda.is_available():
model.cuda()
model.device = torch.device('cuda:0')
else:
model.device = torch.device('cpu')
model.train()
optimizer = AdamW(model.parameters(), lr=0.001, weight_decay=0.0)
bdgl, features, label = next(iter(train_loader))
recursive_to((bdgl, features, label), model.device)
for _ in tqdm(range(200)):
optimizer.zero_grad()
output = model(bdgl, features)
loss = model.loss_fxn(output, label)
if loss < 1e-4:
break
loss.backward()
optimizer.step()
else:
tqdm.write(f'Loss: {loss}')
self.fail("Didn't memorize minibatch")
mlp = MLP(dropout=args.dropout)
optimizer = optim.Adam(itertools.chain(model.parameters(), mlp.parameters()),
lr=args.lr,
weight_decay=args.weight_decay)
creterion = torch.nn.TripletMarginLoss(margin=args.margin, p=2)
CE = torch.nn.CrossEntropyLoss()
loss_list = RunningAvg(window_size=200)
loss_list_CE = RunningAvg(window_size=200)
acc_list = RunningAvg(window_size=200)
loss_by_iter = []
if args.cuda:
device = torch.device(args.cuda_device)
model.cuda(device)
mlp.cuda(device)
def train_step(epoch, loss_save):
for _, data_train_group in tqdm(enumerate(dataloader_train),
desc='Training',
total=len(dataloader_train)):
# pass three times first, then back propagate the loss
model.train()
mlp.train()
for data_train in data_train_group:
vector3 = []
regularization = 0
for data_train_item in data_train:
# and/or children: [[1,2],[3,4]]
def execute(params, budget=None, max_epoch=243, device='cpu', seed=42):
np.random.seed(seed)
torch.manual_seed(seed)
if device == "cuda":
torch.cuda.manual_seed(seed)
# Load data
if params['dataset'] == "cora":
adj, features, labels, idx_train, idx_val, idx_test = load_data(
dataset=params['dataset'], train_percent=0.052)
if params['dataset'] == "citeseer":
adj, features, labels, idx_train, idx_val, idx_test = load_citeseer(
train_percent=0.036)
# Model and optimizer
model = GCN(nfeat=features.shape[1],
nhid=params['hidden'],
nclass=labels.max().item() + 1,
dropout=params['dropout'])
optimizer = optim.Adam(model.parameters(),
lr=params['lr'],
weight_decay=params['weight_decay'])
if device == "cuda":
model.cuda()
features = features.cuda()
adj = adj.cuda()
labels = labels.cuda()
idx_train = idx_train.cuda()
idx_val = idx_val.cuda()
idx_test = idx_test.cuda()
# train model
if device == "cuda":
start = torch.cuda.Event(enable_timing=True)
end = torch.cuda.Event(enable_timing=True)
start.record()
else:
t1 = time.time_ns()
model.train()
num_epoch = int(budget) if budget != None else max_epoch
for epoch in range(num_epoch):
optimizer.zero_grad()
output = model(features, adj)
loss_train = F.nll_loss(output[idx_train], labels[idx_train])
acc_train = accuracy(output[idx_train], labels[idx_train])
loss_train.backward()
optimizer.step()
# evaluation
model.eval()
output = model(features, adj)
loss_val = F.nll_loss(output[idx_val], labels[idx_val])
acc_val = accuracy(output[idx_val], labels[idx_val])
if device == "cuda":
end.record()
torch.cuda.synchronize()
total_time = start.elapsed_time(end) / 1e3
sys.stdout.flush()
acc_val = acc_val.item()
else:
t2 = time.time_ns()
total_time = (t2 - t1) / 1e9
print()
print(
f"dataset={params['dataset']}, num_epoch={num_epoch}, device={next(model.parameters()).device}"
)
print("Validation results:", "loss= {:.4f}".format(loss_val.item()),
"accuracy= {:.4f}".format(acc_val))
print("Total training time: {:.4f} sec".format(total_time))
return 1 - acc_val
dropout=args.dropout,
nnz=adj._nnz())
optimizer = optim.Adam(model.parameters(),
lr=args.lr,
weight_decay=args.weight_decay)
gamma = 0
eta = 10**-6 #should be decreasing with epochs
print('!!!!!!!!!!!!CHECK!!!!!!!!!!!!')
print('save (location) of plots')
print('Gamma:', gamma)
print('Eta:', eta)
print('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')
if args.cuda:
model.cuda()
features = features.cuda()
adj = adj.cuda()
labels = labels.cuda()
idx_train = idx_train.cuda()
idx_val = idx_val.cuda()
idx_test = idx_test.cuda()
#parameters
ew_mean = nn.Parameter(torch.randn(adj._nnz()))
ew_std = nn.Parameter(torch.ones(adj._nnz()))
adj_indices = adj._indices()
adj_size = adj.size()
# Load data
#adj, features, labels, idx_train, idx_val, idx_test = load_data()
adj, A_tilde, adj_sct1, adj_sct2, adj_sct4, features, labels, idx_train, idx_val, idx_test = load_citation(
args.dataset, args.normalization, args.cuda)
# Model and optimizer
model = GCN(nfeat=features.shape[1],
para3=args.hid1,
para4=args.hid2,
nclass=labels.max().item() + 1,
dropout=args.dropout,
smoo=args.smoo)
PATH = "state_dict_model.pt"
model.load_state_dict(torch.load(PATH))
if args.cuda:
model = model.cuda()
features = features.cuda()
A_tilde = A_tilde.cuda()
adj = adj.cuda()
labels = labels.cuda()
idx_train = idx_train.cuda()
idx_val = idx_val.cuda()
idx_test = idx_test.cuda()
optimizer = optim.Adam(model.parameters(),
lr=args.lr,
weight_decay=args.weight_decay)
scheduler = StepLR(optimizer, step_size=50, gamma=0.9)
def test():
def train():
acc_test_list = []
for ii in range(args.run_time):
#seed = gen_seeds()
seed = args.seed
print("dataset:{}, epochs:{}, weight_decay:{},lr:{},dropout:{},seed:{}, alpha:{}, features_perturbation: rate1:{},lambda1:{}; adj_pertubation: rate2:{},lambda2:{}".format(
args.dataset, args.epochs, args.weight_decay, args.lr, args.dropout, seed, args.alpha, args.rate1, args.lambda1, args.rate2,args.lambda2))
args.cuda = not args.no_cuda and torch.cuda.is_available()
np.random.seed(seed)
torch.manual_seed(seed)
if args.cuda:
torch.cuda.manual_seed(seed)
#torch.cuda.manual_seed(seed)
if args.dataset == "R8":
adj, features, labels, idx_train, idx_val, idx_test = load_corpus(args.dataset, args.normalization,args.cuda)
else:
adj, features, labels, idx_train, idx_val, idx_test, indices = load_citation(args.dataset, args.normalization, args.cuda)
model = GCN(nfeat=features.shape[1],
nhid=args.hidden,
nclass=labels.max().item() + 1,
dropout=args.dropout)
if args.cuda:
model.cuda()
features = features.cuda()
adj = adj.cuda()
labels = labels.cuda()
idx_train = idx_train.cuda()
idx_val = idx_val.cuda()
idx_test = idx_test.cuda()
optimizer = optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.weight_decay)
global x,y,z
x,y,z=0,0,0
win = vis.line(X=np.array([x]),
Y=np.array([y]),
opts=dict(title='loss_CE'))
global b_1,b_2
b_0,b_1,b_2=0,0,0
win_b0 = vis.line(X=np.array([x]),
Y=np.array([b_0]),
opts=dict(title='b'))
def train(epoch):
t = time.time()
model.train()
optimizer.zero_grad()
output = model(features, adj)
output = F.log_softmax(output, dim=1)
loss_CE = F.nll_loss(output[idx_train], labels[idx_train])
#l2_reg = sum(torch.sum(param ** 2) for param in model.reg_params)
acc_train = accuracy(output[idx_train], labels[idx_train])
x = epoch
y = loss_CE.detach().cpu().numpy()
vis.line(X=np.array([x]),
Y=np.array([y]),
win=win,
update='append')
#loss_train = loss_CE + args.weight_decay /2 *l2_reg
loss_train = loss_CE
loss_train.backward()
optimizer.step()
if not args.fastmode:
model.eval()
output = model(features, adj)
output = F.log_softmax(output, dim=1)
loss_val = F.nll_loss(output[idx_val], labels[idx_val])
acc_val = accuracy(output[idx_val], labels[idx_val])
if ii == 0:
if epoch%10==0:
print('Epoch: {:04d}'.format(epoch + 1),
'loss_train: {:.4f}'.format(loss_train.item()),
'loss_CE: {:.4f}'.format(loss_CE.item()),
'acc_train: {:.4f}'.format(acc_train.item()),
#'loss_0: {:.4f}'.format(loss_0.item()),
# 'loss_fx: {:.4f}'.format(loss_fx.item()),
# 'loss_logfx: {:.4f}'.format(loss_logfx.item()),
# 'acc_train: {:.4f}'.format(acc_train.item()),
'loss_val: {:.4f}'.format(loss_val.item()),
'acc_val: {:.4f}'.format(acc_val.item()),
'time: {:.4f}s'.format(time.time() - t))
return loss_val.item()
def test():
model.eval()
output = model(features,adj)
output = F.log_softmax(output, dim=1)
loss_test = F.nll_loss(output[idx_test], labels[idx_test])
acc_test = accuracy(output[idx_test], labels[idx_test])
print("Test set results:",
"loss_test= {:.4f}".format(loss_test.item()),
"accuracy= {:.4f}".format(acc_test.item()))
return acc_test
t_total = time.time()
loss_values = []
# bad_counter = 0
# best = args.epochs+1
# best_epoch=0
# for epoch in range(args.epochs):
#
# loss_values.append(train(epoch))
# torch.save(model.state_dict(), './checkpoints/{}/{}.pkl'.format(args.dataset, epoch))
# if loss_values[-1] < best:
# best = loss_values[-1]
# best_epoch = epoch
# bad_counter = 0
# else:
# bad_counter += 1
# if bad_counter == args.patience:
# break
# files = glob.glob('./checkpoints/{}/*.pkl'.format(args.dataset)) # =[/checkpoints/{}/{}/0.pkl]
# for file in files:
# epoch_nb = int(file.split('.')[-2].split('/')[-1])
# if epoch_nb < best_epoch:
# os.remove(file)
# files = glob.glob('./checkpoints/{}/*.pkl'.format(args.dataset))
# for file in files:
# epoch_nb = int(file.split('.')[-2].split('/')[-1])
# if epoch_nb > best_epoch:
# os.remove(file)
# print("Optimization Finished!")
# print("Total time elapsed: {:.4f}s".format(time.time() - t_total))
#
# # Testing
# print('Loading {}th epoch'.format(best_epoch))
# model.load_state_dict(torch.load('./checkpoints/{}/{}.pkl'.format(args.dataset, best_epoch)))
for epoch in range(args.epochs):
loss_values.append(train(epoch))
if epoch>args.early_stop and loss_values[-1] > np.mean(loss_values[-(args.early_stop+1):-1]):
print("Early stopping...")
break
#train(epoch)
print("Optimization Finished!")
print("Total time elapsed: {:.4f}s".format(time.time() - t_total))
acc_test = test()
acc_test_list.append(acc_test)
acc_test = acc_test.view(1, 1)
acc_test = acc_test.cpu().numpy()
with open("./results/{}/{}.txt".format(args.dataset, args.dataset), 'a') as f:
# f.write("不丢弃dropout,加上b= {:.07f}×(output-output_1),epoch : {:04d},weight_decacy={},系数lam ={:.07f}".format
f.write("dataset{} epoch : {:04d},weight_decacy={}, lr{},seed{},dropout{},features_perturbation: rate1:{},lambda1:{}; adj_pertubation: rate2:{},lambda2:{}".format
(args.dataset, args.epochs, args.weight_decay, args.lr, seed, args.dropout,args.rate1,args.lambda1,args.rate2,args.lambda2))
np.savetxt(f, acc_test, fmt="%.6f")
acc_test_list = torch.FloatTensor(acc_test_list)
acc_test_std = torch.std(acc_test_list)
avg_test = torch.mean(acc_test_list)
avg_test = avg_test.view(1, 1)
avg_test = avg_test.cpu().numpy()
acc_test_std = acc_test_std.view(1, 1)
acc_test_std = acc_test_std.cpu().numpy()
print("总共做类{}次实验，平均值为：{:.04f}".format(args.run_time, avg_test.item()))
print("总共做类{}次实验，误差值为：{:.04f}".format(args.run_time,acc_test_std.item()))
with open("./results/{}/{}.txt".format(args.dataset, args.dataset), 'a') as f:
f.write("总共做类{}次实验，平均值为：{:.04f}\n".format(args.run_time, avg_test.item()))
f.write("总共做类{}次实验，误差值为：{:.04f}\n".format(args.run_time, acc_test_std.item()))
nclass=labels.max().item() + 1,
dropout=0)
student_params = list(model_S.parameters())
teacher_params = list(model_T.parameters())
for param in teacher_params:
param.requires_grad = False
optimizer_S = optim.Adam(model_S.parameters(),
lr=args.lr,
weight_decay=args.weight_decay)
optimizer_T = WeightEMA(teacher_params, student_params, alpha=args.alpha)
if args.cuda:
model_S.cuda(args.gpu)
model_T.cuda(args.gpu)
features = features.cuda(args.gpu)
adj_T = adj_T.cuda(args.gpu)
adj_S = adj_S.cuda(args.gpu)
labels = labels.cuda(args.gpu)
pseudo_labels = pseudo_labels.cuda(args.gpu)
idx_train = idx_train.cuda(args.gpu)
idx_val = idx_val.cuda(args.gpu)
idx_test = idx_test.cuda(args.gpu)
idx_self = idx_self.cuda(args.gpu)
features, labels, pseudo_labels = Variable(features), Variable(
labels), Variable(pseudo_labels)
def get_validation_loss(no_cuda=True,
fastmode=True,
seed=42,
epochs=50,
lr=0.01,
weight_decay=5e-4,
hidden_dim_gcn=32,
output_dim=16,
dropout_prop=0.5,
hidden_dim_fc=32,
verbose=False
):
# Training settings
# help='Weight decay (L2 loss on parameters).')
cuda = not no_cuda and torch.cuda.is_available()
np.random.seed(seed)
torch.manual_seed(seed)
if cuda:
torch.cuda.manual_seed(seed)
# Load data
(training_features,
validation_features,
training_adj,
validation_adj,
training_labels,
validation_labels) = load_all_ligands()
# Model and optimizer
model = GCN(init_dim=11,
hidden_dim_gcn=hidden_dim_gcn,
output_dim=output_dim,
dropout_prop=dropout_prop,
hidden_dim_fc=hidden_dim_fc)
optimizer = optim.Adam(model.parameters(),
lr=lr, weight_decay=weight_decay)
loss = nn.BCELoss()
if cuda:
model.cuda()
training_features = [features.cuda() for features in training_features]
validation_features = [features.cuda() for features in validation_features]
training_adj = [adj.cuda() for adj in training_adj]
validation_adj = [adj.cuda() for adj in validation_adj]
training_adj = [adj.cuda() for adj in training_adj]
validation_adj = [adj.cuda() for adj in validation_adj]
training_labels = [labels.cuda() for labels in training_labels]
validation_labels = [labels.cuda() for labels in validation_labels]
def train(epoch):
model.train()
for i in range(len(training_labels)):
optimizer.zero_grad()
output = model(training_features[i], training_adj[i])
loss_train = loss(output, training_labels[i])
#acc_train = accuracy(output, labels)
if i % 32 == 0:
loss_train.backward()
optimizer.step()
if verbose:
print('Epoch: {:04d}'.format(epoch+1),
'loss_train: {:.4f}'.format(loss_train.item()))
def validate():
model.eval()
for i in range(len(validation_labels)):
output = model(validation_features[i], validation_adj[i])
loss_validation = loss(output, validation_labels[i])
#acc_validation = accuracy(output, labels)
if verbose:
print('Epoch: {:04d}'.format(epoch+1),
'loss_validation: {:.4f}'.format(loss_validation.item()))
return loss_validation.item()
# Train model
t_total = time.time()
for epoch in range(epochs):
train(epoch)
if not fastmode:
# Evaluate validation set performance separately,
# deactivates dropout during validation run.
validate()
if verbose:
print("Optimization Finished!")
print("Total time elapsed: {:.4f}s".format(time.time() - t_total))
# Testing
return validate()
def experimenter(data_name='cora',
train_ratio=0.03,
cuda=True,
random_seed=42,
hidden=16,
dropout_ratio=0.5,
learning_rate=0.01,
weight_decay=5e-4,
num_epochs=65,
early_stopping=30,
task='classification',
public_splitting=False):
# helper function to run epxeriment
if data_name in ['cora', 'citeseer', 'pubmed']:
print("Loading Classification Datasets")
Tmat, eadj, edge_name, edge_feature_dict, adj, features, edge_features, labels, idx_train, idx_val, idx_test = tqdm(
load_data(data_name=data_name,
train_ratio=train_ratio,
public_splitting=public_splitting))
model = GCN(nfeat_v=features.shape[1],
nfeat_e=edge_features.shape[1],
nhid=hidden,
nclass=labels.max().item() + 1,
dropout=dropout_ratio)
ssl_agent_n = PairwiseDistance(adj,
features,
nhid=args.hidden,
cuda=cuda,
node=True)
ssl_agent_e = PairwiseDistance(eadj,
edge_features,
nhid=args.hidden,
cuda=cuda,
node=False)
else:
ValueError("The input data is not supported! ")
print(">" * 100)
print("Loaded and preprocessed the graph data! ")
print(">" * 100)
optimizer = optim.Adam(model.parameters(),
lr=learning_rate,
weight_decay=weight_decay)
np.random.seed(random_seed)
torch.manual_seed(random_seed)
if cuda:
torch.cuda.manual_seed(random_seed)
torch.set_default_tensor_type('torch.cuda.FloatTensor')
if cuda:
model.cuda()
Tmat = Tmat.cuda()
eadj = eadj.cuda()
adj = adj.cuda()
features = features.cuda()
edge_features = edge_features.cuda()
labels = labels.cuda()
idx_train = idx_train.cuda()
idx_val = idx_val.cuda()
idx_test = idx_test.cuda()
# pooling = pooling.cuda()
# node_count = node_count.cuda()
if task == "classification":
criteria = F.nll_loss
acc_measure = accuracy
elif task == "regression":
criteria = torch.nn.L1Loss
acc_measure = RMSELoss
# ---------------------------------------
# training function
# ---------------------------------------
# count_time = 0
def train(epoch):
t = time.time()
model.train()
optimizer.zero_grad()
output, n_feature, e_feature = model(features, edge_features, eadj,
adj, Tmat, task)
loss_n = ssl_agent_n.classification_loss(n_feature)
loss_e = ssl_agent_e.classification_loss(e_feature)
loss_train = criteria(output[idx_train], labels[idx_train])
loss_train = loss_train + 0.1 * loss_n + 0.1 * loss_e
acc_train = acc_measure(output[idx_train], labels[idx_train])
loss_train.backward()
optimizer.step()
if not args.fastmode:
# Evaluate validation set performance separately,
# deactivates dropout during validation run.
model.eval()
output, _, _ = model(features, edge_features, eadj, adj, Tmat,
task)
loss_val = criteria(output[idx_val], labels[idx_val])
acc_val = acc_measure(output[idx_val], labels[idx_val])
print('Epoch: {:04d}'.format(epoch + 1),
'loss_train: {:.4f}'.format(loss_train.item()),
'acc_train: {:.4f}'.format(acc_train.item()),
'loss_val: {:.4f}'.format(loss_val.item()),
'acc_val: {:.4f}'.format(acc_val.item()),
'time: {:.4f}s'.format(time.time() - t))
return loss_val.item()
# -------------------------------------------
# testing function
# -------------------------------------------
def test():
model.eval()
output, _, _ = model(features, edge_features, eadj, adj, Tmat, task)
loss_test = criteria(output[idx_test], labels[idx_test])
acc_test = acc_measure(output[idx_test], labels[idx_test])
print("Test set results:", "loss= {:.4f}".format(loss_test.item()),
"accuracy= {:.4f}".format(acc_test.item()))
return acc_test.item()
def pretrain(epoch):
t = time.time()
model.train()
optimizer.zero_grad()
output, node_f, edge_f = model(features, edge_features, eadj, adj,
Tmat, task)
loss_n = ssl_agent_n.classification_loss(node_f)
loss_e = ssl_agent_e.classification_loss(edge_f)
# loss_train = criteria(output[idx_train], labels[idx_train])
loss_train = loss_n + loss_e
acc_train = acc_measure(output[idx_train], labels[idx_train])
loss_train.backward()
optimizer.step()
if not args.fastmode:
# Evaluate validation set performance separately,
# deactivates dropout during validation run.
model.eval()
output, _, _ = model(features, edge_features, eadj, adj, Tmat,
task)
loss_val = criteria(output[idx_val], labels[idx_val])
acc_val = acc_measure(output[idx_val], labels[idx_val])
print('Epoch: {:04d}'.format(epoch + 1),
'loss_train: {:.4f}'.format(loss_train.item()),
'acc_train: {:.4f}'.format(acc_train.item()),
'loss_val: {:.4f}'.format(loss_val.item()),
'acc_val: {:.4f}'.format(acc_val.item()),
'time: {:.4f}s'.format(time.time() - t))
return loss_val.item()
# Train model
t_total = time.time()
val_watch = []
pre_val_watch = []
input_idx_train = idx_train
# for epoch in range(50):
# pre_val_watch.append(pretrain(epoch))
# test()
# if epoch > early_stopping and pre_val_watch[-1] > np.mean(pre_val_watch[-(early_stopping + 1):-1]):
# print("Early stopping...")
# break
for epoch in range(num_epochs):
# val_adj, val_fea = ssl_agent.transform_data()
val_watch.append(train(epoch))
test()
if epoch > early_stopping and val_watch[-1] > np.mean(
val_watch[-(early_stopping + 1):-1]):
print("Early stopping...")
break
print("Optimization Finished!")
print("Total time elapsed: {:.4f}s".format(time.time() - t_total))
print("Printing the weights : ")
return test()
args = parser.parse_args()
n_anchor = 1000
n_bits = args.n_bits
n_class = 21
n_epoch = 10
topk = 15
# dataset: 'cifar10', 'nuswide', 'ImageNet', 'sun'
dataset = 'nuswide'
dset = load_data(dataset)
meta = torch.load('nuswide_2000_32_0.4454_0.5912')
anchor = meta['anchor']
gcn = GCN(500, n_bits, n_class, meta['anchor_affnty'], 40)
gcn.load_state_dict(meta['state_dict'])
gcn.cuda()
test_loader = data.DataLoader(dataset=db(dset.testdata, dset.testlabel),
batch_size=100,
shuffle=False,
num_workers=4)
tH = []
gcn.eval()
for images, _ in test_loader:
in_aff, out_aff = rbf_affnty(images, anchor, topk=topk)
images = Variable(images).cuda()
in_aff = Variable(in_aff).cuda()
out_aff = Variable(out_aff).cuda()
out, _ = gcn(images, in_aff, out_aff)
tH.append(out.data.cpu().numpy())
# print('idx_train的维度:',idx_train.shape) # 140*1 训练节点标签
# print('idx_val的维度:',idx_val.shape) # 300*1 验证节点标签
# print('idx_test的维度:',idx_test.shape) # 1000*1 测试节点标签
# Model and optimizer
model = GCN(nfeat=features.shape[1],
nhid=args.hidden,
nclass=labels.max().item() + 1,
dropout=args.dropout)
optimizer = optim.Adam(model.parameters(),
lr=args.lr,
weight_decay=args.weight_decay)
# 数据写入cuda，便于后续加速
if args.cuda:
model.cuda() # . cuda()会分配到显存里（如果gpu可用）
features = features.cuda()
adj = adj.cuda()
labels = labels.cuda()
idx_train = idx_train.cuda()
idx_val = idx_val.cuda()
idx_test = idx_test.cuda()
def train(epoch):
t = time.time() # 返回当前时间
model.train()
optimizer.zero_grad()
# optimizer.zero_grad()意思是把梯度置零，也就是把loss关于weight的导数变成0.
# pytorch中每一轮batch需要设置optimizer.zero_gra
output = model(features, adj)
features = features.cuda()
adj = adj.cuda()
labels = labels.cuda()
idx_train = idx_train.cuda()
idx_val = idx_val.cuda()
idx_test = idx_test.cuda()
features, adj, labels = Variable(features), Variable(adj), Variable(labels)
#net = nn.DataParallel(net, device_ids=range(1))
#net.load_state_dict(torch.load(opt.modelIn))
optimizer = optim.Adam(net.parameters(),
lr=0.01, weight_decay=5e-4)
net.cuda()
for epoch_n in range(200):
train(epoch_n)
loss_f = nn.CrossEntropyLoss()
net.eval()
output = net(features, adj)
output = F.log_softmax(output, dim =1)
loss_test = F.nll_loss(output[idx_test], labels[idx_test])
acc_test = accuracy(output[idx_test], labels[idx_test])
print("Test set results:",
"loss= {:.4f}".format(loss_test.data[0]),
"accuracy= {:.4f}".format(acc_test.data[0]))
for c in opt.c:
#print (type(adj), type(features))
acc, avg_distort = acc_under_attack(features,adj,labels, net, c, attack_cw)
def train_gcn(dataset,
test_ratio=0.5,
val_ratio=0.2,
seed=1,
n_hidden=64,
n_epochs=200,
lr=1e-2,
weight_decay=5e-4,
dropout=0.5,
use_embs=False,
verbose=True,
cuda=False):
data = dataset.get_data()
# train text embs
if use_embs:
pad_ix, n_tokens, matrix, pretrained_embs = data['features']
if pretrained_embs is not None:
pretrained_embs = torch.FloatTensor(pretrained_embs)
features = torch.LongTensor(matrix)
else:
pad_ix = None
n_tokens = None
pretrained_embs = None
features = torch.FloatTensor(data['features'])
labels = torch.LongTensor(data['labels'])
n = len(data['ids'])
train_mask, val_mask, test_mask = get_masks(n,
data['main_ids'],
data['main_labels'],
test_ratio=test_ratio,
val_ratio=val_ratio,
seed=seed)
train_mask = torch.BoolTensor(train_mask)
val_mask = torch.BoolTensor(val_mask)
test_mask = torch.BoolTensor(test_mask)
if cuda:
torch.cuda.set_device("cuda:0")
features = features.cuda()
labels = labels.cuda()
train_mask = train_mask.cuda()
val_mask = val_mask.cuda()
test_mask = test_mask.cuda()
g = DGLGraph(data['graph'])
g = dgl.transform.add_self_loop(g)
n_edges = g.number_of_edges()
degs = g.in_degrees().float()
norm = torch.pow(degs, -0.5)
norm[torch.isinf(norm)] = 0
if cuda:
norm = norm.cuda()
g.ndata['norm'] = norm.unsqueeze(1)
if use_embs:
if pretrained_embs is not None:
in_feats = 100
else:
in_feats = 64
else:
in_feats = features.shape[1]
# + 1 for unknown class
n_classes = data['n_classes'] + 1
model = GCN(g,
in_feats=in_feats,
n_hidden=n_hidden,
n_classes=n_classes,
activation=F.relu,
dropout=dropout,
use_embs=use_embs,
pretrained_embs=pretrained_embs,
pad_ix=pad_ix,
n_tokens=n_tokens)
if cuda:
model.cuda()
loss_fcn = torch.nn.CrossEntropyLoss()
# use optimizer
optimizer = torch.optim.Adam(model.parameters(),
lr=lr,
weight_decay=weight_decay)
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer,
mode='min',
factor=0.9,
patience=20,
min_lr=1e-10)
best_f1 = -100
# initialize graph
dur = []
for epoch in range(n_epochs):
model.train()
if epoch >= 3:
t0 = time.time()
# forward
mask_probs = torch.empty(features.shape).uniform_(0, 1)
if cuda:
mask_probs = mask_probs.cuda()
mask_features = torch.where(mask_probs > 0.2, features,
torch.zeros_like(features))
logits = model(mask_features)
loss = loss_fcn(logits[train_mask], labels[train_mask])
optimizer.zero_grad()
loss.backward()
optimizer.step()
if epoch >= 3:
dur.append(time.time() - t0)
f1 = evaluate(model, features, labels, val_mask)
scheduler.step(1 - f1)
if f1 > best_f1:
best_f1 = f1
torch.save(model.state_dict(), 'best_model.pt')
if verbose:
print("Epoch {:05d} | Time(s) {:.4f} | Loss {:.4f} | F1 {:.4f} | "
"ETputs(KTEPS) {:.2f}".format(epoch, np.mean(dur),
loss.item(), f1,
n_edges / np.mean(dur) / 1000))
model.load_state_dict(torch.load('best_model.pt'))
f1 = evaluate(model, features, labels, test_mask)
if verbose:
print()
print("Test F1 {:.2}".format(f1))
return f1
torch.manual_seed(args.seed) # 为CPU设置随机种子用于生成随机数，以使得结果是确定的
if args.cuda:
torch.cuda.manual_seed(args.seed) # 为GPU设置随机种子
# 加载数据
adj, features, labels, idx_train, idx_val, idx_test = load_data()
# 模型和优化器
model = GCN(nfeat=features.shape[1],
nhid=args.hidden,
nclass=labels.max().item() + 1,
dropout=args.dropout,
meta_size=args.meta_size)
optimizer = optim.Adam(model.parameters(),
lr=args.lr,
weight_decay=args.weight_decay)
if args.cuda: # 如果使用GUP则执行这里，数据写入cuda，便于后续加速
model.cuda() # 模型放到GPU上跑
features = features.cuda()
adj = adj.cuda()
labels = labels.cuda()
idx_train = idx_train.cuda()
idx_val = idx_val.cuda()
idx_test = idx_test.cuda()
def train(epoch):
"""
定义训练函数
:param epoch: 训练轮数
"""
t = time.time()
model.train()
def train_run(img_name):
adj, features, labels, idx_train, idx_val, idx_test, rgbxy = load_data(
dataset=img_name)
# Model and optimizer
model = GCN(nfeat=features.shape[1],
nhid=80,
nclass=labels.max().item() + 1,
dropout=0.5)
optimizer = optim.Adam(model.parameters(), lr=0.01, weight_decay=5e-4)
print("a\n")
if args.cuda:
model.cuda()
features = features.cuda()
adj = adj.cuda()
labels = labels.cuda()
idx_train = idx_train.cuda()
idx_val = idx_val.cuda()
idx_test = idx_test.cuda()
def train(epoch):
t = time.time()
model.train()
optimizer.zero_grad()
output = model(features, adj)
loss_train1 = F.nll_loss(output[idx_train], labels[idx_train])
'''
print(labels.shape)
#print(output.shape)
c = torch.where(labels.cpu() == 0,torch.ones(labels.shape),torch.zeros(labels.shape))
print(c.shape)
background_points = torch.sum( c )
total_points = labels.shape[0]
foreground_points = total_points - background_points
l = labels[idx_train].cpu()
ba = l.shape[0]
deep = output.shape[1]
labels_one_hot = torch.zeros((ba,deep))
#labels_one_hot[:,0] = torch.where(l == 0, torch.ones(ba)/background_points, torch.zeros(ba))
#labels_one_hot[:,0] = torch.where(l == 0, torch.ones(ba)*total_points/background_points, torch.zeros(ba))
labels_one_hot[:,0] = torch.where(l == 0, torch.ones(ba), torch.zeros(ba))
for i in range(1,deep):
#labels_one_hot[:,i] = torch.where(l == i, torch.ones(ba)/foreground_points*foreground_inputs, torch.zeros(ba))
#labels_one_hot[:,i] = torch.where(l == i, torch.ones(ba)*total_points/foreground_points, torch.zeros(ba))
labels_one_hot[:,i] = torch.where(l == i, torch.ones(ba), torch.zeros(ba))
loss_func = torch.nn.BCELoss()
#print(output[idx_train].shape)
#print(labels_one_hot.shape)
soft = torch.nn.Softmax(dim=1)
loss_train1 = loss_func( soft(output[idx_train]).cpu(), labels_one_hot)
'''
'''
rowsum = torch.sum(output[idx_train],1)
r_inv = torch.pow(rowsum, -1)
r_inv[torch.isinf(r_inv)] = 0
r_mat_inv = torch.diag(r_inv)
mx = torch.mm(r_mat_inv, output[idx_train])
loss_train1 = loss_func( mx.cpu(), labels_one_hot)
'''
#print("b= {}, f = {}".format(background_points,foreground_points))
l = output[idx_test].shape[0]
W = torch.zeros((l, l, rgbxy.shape[1])).cuda()
ones = torch.unsqueeze(torch.ones((l, 5)), 0)
rgbxy_2d = rgbxy[idx_test]
rgbxy_3d = []
for i in range(l):
rgbxy_3d.append(rgbxy_2d)
rgbxy_3d = tuple(rgbxy_3d)
rgbxy_3d = torch.stack(rgbxy_3d, dim=1).cuda()
# ones_t = ones.transpose(ones, 1,2)
'''
for i in range(l):
for j in range(l):
W[i][j] = rgbxy[idx_test[i]]-rgbxy[idx_test[j]]
'''
W = rgbxy_3d.transpose(0, 1) - rgbxy_3d
#'''
sigma = args.sigma
W = torch.exp(-torch.norm(W, dim=2) / 2 / sigma)
d = torch.sum(W, dim=1)
n_cut = 1
for k in range(output.shape[1]):
s = output[idx_test, k]
n_cut = n_cut + torch.mm(torch.mm(torch.unsqueeze(
s, 0), W), torch.unsqueeze(1 - s, 1)) / (torch.dot(d, s))
lamda = args.lamda
print(n_cut)
loss_train = loss_train1.cuda() + lamda * n_cut
acc_train = accuracy(output[idx_train], labels[idx_train])
loss_train.backward()
optimizer.step()
if not False:
# Evaluate validation set performance separately,
# deactivates dropout during validation run.
model.eval()
output = model(features, adj)
#loss_val = F.nll_loss(output[idx_val], labels[idx_val])
loss_val = loss_train1
acc_val = accuracy(output[idx_val], labels[idx_val])
print('Epoch: {:04d}'.format(epoch + 1),
'loss_train: {:.4f}'.format(loss_train.item()),
'acc_train: {:.4f}'.format(acc_train.item()),
'loss_val: {:.4f}'.format(loss_val.item()),
'acc_val: {:.4f}'.format(acc_val.item()),
'time: {:.4f}s'.format(time.time() - t))
def test():
model.eval()
output = model(features, adj)
loss_test = F.nll_loss(output[idx_test], labels[idx_test])
acc_test = accuracy(output[idx_test], labels[idx_test])
print("Test set results:", "loss= {:.4f}".format(loss_test.item()),
"accuracy= {:.4f}".format(acc_test.item()))
#np.save(img_name+'predict',output.detach().cpu().numpy())
print("save prediction!---" + img_name + 'predict')
with open("retport.txt", 'a') as f:
f.write(img_name + "\t")
f.write("test acc:" + str(acc_test) + "\n")
# Train model
t_total = time.time()
for epoch in range(200):
train(epoch)
print("Optimization Finished!")
print("Total time elapsed: {:.4f}s".format(time.time() - t_total))
# Testing
test()
from postprocess import run
run(img_name)
