def test_one_shot(args, model, test_samples=5000, partition='test'):
io = io_utils.IOStream('checkpoints/' + args.exp_name + '/run.log')
io.cprint('\n**** TESTING WITH %s ***' % (partition,))
loader = generator.Generator(args.dataset_root, args, partition=partition, dataset=args.dataset)
[enc_nn, metric_nn, softmax_module] = model
enc_nn.eval()
metric_nn.eval()
correct = 0
total = 0
iterations = int(test_samples/args.batch_size_test)
for i in range(iterations):
data = loader.get_task_batch(batch_size=args.batch_size_test, n_way=args.test_N_way,
num_shots=args.test_N_shots, unlabeled_extra=args.unlabeled_extra)
[x, labels_x_cpu, _, _, xi_s, labels_yi_cpu, oracles_yi, hidden_labels] = data
if args.cuda:
xi_s = [batch_xi.cuda() for batch_xi in xi_s]
labels_yi = [label_yi.cuda() for label_yi in labels_yi_cpu]
oracles_yi = [oracle_yi.cuda() for oracle_yi in oracles_yi]
hidden_labels = hidden_labels.cuda()
x = x.cuda()
else:
labels_yi = labels_yi_cpu
xi_s = [Variable(batch_xi) for batch_xi in xi_s]
labels_yi = [Variable(label_yi) for label_yi in labels_yi]
oracles_yi = [Variable(oracle_yi) for oracle_yi in oracles_yi]
hidden_labels = Variable(hidden_labels)
x = Variable(x)
# Compute embedding from x and xi_s
z = enc_nn(x)[-1]
zi_s = [enc_nn(batch_xi)[-1] for batch_xi in xi_s]
# Compute metric from embeddings
output, out_logits = metric_nn(inputs=[z, zi_s, labels_yi, oracles_yi, hidden_labels])
output = out_logits
y_pred = softmax_module.forward(output)
y_pred = y_pred.data.cpu().numpy()
y_pred = np.argmax(y_pred, axis=1)
labels_x_cpu = labels_x_cpu.numpy()
labels_x_cpu = np.argmax(labels_x_cpu, axis=1)
for row_i in range(y_pred.shape[0]):
if y_pred[row_i] == labels_x_cpu[row_i]:
correct += 1
total += 1
if (i+1) % 100 == 0:
io.cprint('{} correct from {} \tAccuracy: {:.3f}%)'.format(correct, total, 100.0*correct/total))
io.cprint('{} correct from {} \tAccuracy: {:.3f}%)'.format(correct, total, 100.0*correct/total))
io.cprint('*** TEST FINISHED ***\n'.format(correct, total, 100.0 * correct / total))
enc_nn.train()
metric_nn.train()
return 100.0 * correct / total
help='Decreasing the learning rate every x iterations')
args = parser.parse_args()
def _init_():
if not os.path.exists('checkpoints'):
os.makedirs('checkpoints')
if not os.path.exists('checkpoints/'+args.exp_name):
os.makedirs('checkpoints/'+args.exp_name)
if not os.path.exists('checkpoints/'+args.exp_name+'/'+'models'):
os.makedirs('checkpoints/'+args.exp_name+'/'+'models')
os.system('cp main.py checkpoints'+'/'+args.exp_name+'/'+'main.py.backup')
os.system('cp models/models.py checkpoints' + '/' + args.exp_name + '/' + 'models.py.backup')
_init_()
io = io_utils.IOStream('checkpoints/' + args.exp_name + '/run.log')
io.cprint(str(args))
args.cuda = not args.no_cuda and torch.cuda.is_available()
torch.manual_seed(args.seed)
if args.cuda:
io.cprint('Using GPU : ' + str(torch.cuda.current_device())+' from '+str(torch.cuda.device_count())+' devices')
torch.cuda.manual_seed(args.seed)
else:
io.cprint('Using CPU')
def train_batch(model, data):
[enc_nn, metric_nn, softmax_module] = model
[batch_x, label_x, batches_xi, labels_yi, oracles_yi, hidden_labels] = data
def test_one_shot(args,
fold,
test_root,
model,
test_samples=50,
partition='test',
io_path='run.log'):
io = io_utils.IOStream(io_path)
io.cprint('\n**** TESTING BEGIN ***')
root = test_root
loader = Generator(root, keys=['CN', 'MCI', 'AD'])
[amgnn, softmax_module] = model
amgnn.eval()
correct = 0
total = 0
pre_all = []
pre_all_num = []
real_all = []
iterations = int(test_samples / args.batch_size_test)
for i in range(iterations):
data = loader.get_task_batch(batch_size=args.batch_size_test,
n_way=args.test_N_way,
num_shots=args.test_N_shots,
unlabeled_extra=args.unlabeled_extra,
cuda=args.cuda)
[x_t, labels_x_cpu_t, _, _, xi_s, labels_yi_cpu, oracles_yi] = data
z_clinical, z_mri_feature = x_t[:, 0, 0, 0:args.
clinical_feature_num], x_t[:, :, :,
args.
clinical_feature_num:]
zi_s_clinical = [
batch_xi[:, 0, 0, 0:args.clinical_feature_num] for batch_xi in xi_s
]
zi_s_mri_feature = [
batch_xi[:, :, :, args.clinical_feature_num:] for batch_xi in xi_s
]
adj = compute_adj(z_clinical, zi_s_clinical)
x = x_t
labels_x_cpu = labels_x_cpu_t
if args.cuda:
xi_s = [batch_xi.cuda() for batch_xi in zi_s_mri_feature]
labels_yi = [label_yi.cuda() for label_yi in labels_yi_cpu]
oracles_yi = [oracle_yi.cuda() for oracle_yi in oracles_yi]
x = x.cuda()
else:
labels_yi = labels_yi_cpu
xi_s = [Variable(batch_xi) for batch_xi in zi_s_mri_feature]
labels_yi = [Variable(label_yi) for label_yi in labels_yi]
oracles_yi = [Variable(oracle_yi) for oracle_yi in oracles_yi]
z_mri_feature = Variable(z_mri_feature)
# Compute metric from embeddings
output, out_logits = amgnn(inputs=[
z_clinical, z_mri_feature, zi_s_clinical, xi_s, labels_yi,
oracles_yi, adj
])
output = out_logits
Y = softmax_module.forward(output)
y_pred = softmax_module.forward(output)
y_pred = y_pred.data.cpu().numpy()
y_inter = [list(y_i) for y_i in y_pred]
pre_all_num = pre_all_num + list(y_inter)
y_pred = np.argmax(y_pred, axis=1)
labels_x_cpu = labels_x_cpu.cpu().numpy()
labels_x_cpu = np.argmax(labels_x_cpu, axis=1)
pre_all = pre_all + list(y_pred)
real_all = real_all + list(labels_x_cpu)
for row_i in range(y_pred.shape[0]):
if y_pred[row_i] == labels_x_cpu[row_i]:
correct += 1
total += 1
labels_x_cpu = Variable(torch.cuda.LongTensor(labels_x_cpu))
loss_test = F.nll_loss(Y, labels_x_cpu)
loss_test_f = float(loss_test)
del loss_test
io.cprint('real_label: ' + str(real_all))
io.cprint('pre_all: ' + str(pre_all))
io.cprint('pre_all_num: ' + str(pre_all_num))
io.cprint('{} correct from {} \tAccuracy: {:.3f}%)'.format(
correct, total, 100.0 * correct / total))
io.cprint('*** TEST FINISHED ***\n'.format(correct, total,
100.0 * correct / total))
amgnn.train()
return 100.0 * correct / total, loss_test_f
def test_one_shot(e_stop,
test_cycle,
args,
model,
test_samples=5000,
partition='test'):
"""Function used to perform testing (validation and final test)
Parameters:
e_stop (early_stop): early stop which monitors when we stop training
test_cycle (int): current iteration cycle - used when creating confusion matrix file names
args (Namespace): arguments from argparse
model (list) : contains cnn, gnn and softmax models
test_samples (int) : number of samples to test
partition (string) : train, val or test
Returns:
e_stop (early_stop) : we return the early_stop object which has been updated based upon the test
"""
io = io_utils.IOStream('checkpoints/' + args.exp_name + '/run.log')
io.cprint('\n**** TESTING WITH %s ***' % (partition, ))
loader = generator.Generator(args.dataset_root,
args,
partition=partition,
dataset=args.dataset)
[enc_nn, metric_nn, softmax_module] = model
enc_nn.eval()
metric_nn.eval()
correct = 0
total = 0
iterations = int(test_samples / args.batch_size_test)
true_list = []
predicted_list = []
with open(
os.path.join('datasets', 'compacted_datasets',
'sensor_label_decoder.pickle'), 'rb') as handle:
label_decoder = pickle.load(handle)
sep = '\\'
for temp in range(0, len(label_decoder)):
label_decoder[temp] = label_decoder[temp].rsplit(sep, 1)[1]
for i in range(iterations):
data, labels_dict = loader.get_task_batch(
batch_size=args.batch_size_test)
[x, labels_x_cpu, _, x_global, xi_s, labels_yi_cpu] = data
if args.cuda:
xi_s = [batch_xi.cuda() for batch_xi in xi_s]
labels_yi = [label_yi.cuda() for label_yi in labels_yi_cpu]
x = x.cuda()
else:
labels_yi = labels_yi_cpu
xi_s = [Variable(batch_xi) for batch_xi in xi_s]
labels_yi = [Variable(label_yi) for label_yi in labels_yi]
x = Variable(x)
# Compute embedding from x and xi_s
z = enc_nn(x)
zi_s = [enc_nn(batch_xi) for batch_xi in xi_s]
# Compute metric from embeddings
output, out_logits = metric_nn(inputs=[z, zi_s, labels_yi])
output = out_logits
y_pred = softmax_module.forward(output)
y_pred = y_pred.data.cpu().numpy()
y_pred = np.argmax(y_pred, axis=1)
labels_x_cpu = labels_x_cpu.numpy()
labels_x_cpu = np.argmax(labels_x_cpu, axis=1)
for i in range(0, len(labels_x_cpu)):
true_label = labels_dict[i, labels_x_cpu[i]]
true_list.append(label_decoder[true_label])
predicted_label = labels_dict[i, y_pred[i]]
predicted_list.append(label_decoder[predicted_label])
for row_i in range(y_pred.shape[0]):
if y_pred[row_i] == labels_x_cpu[row_i]:
correct += 1
total += 1
if (i + 1) % 100 == 0:
io.cprint('{} correct from {} \tAccuracy: {:.3f}%)'.format(
correct, total, 100.0 * correct / total))
acc = accuracy_score(true_list, predicted_list)
micro = f1_score(true_list, predicted_list, average='weighted')
macro = f1_score(true_list, predicted_list, average='macro')
e_stop.update(micro, enc_nn, metric_nn)
if partition == 'test' or (partition == 'val' and e_stop.improve):
if partition == 'test':
test_cycle = 999
#Print confusion matrix
conf_mat.conf_mat(true_list, predicted_list, test_cycle, args.train,
args.test)
test_labels = sorted(set(true_list).union(set(predicted_list)))
print(
classification_report(true_list,
predicted_list,
target_names=test_labels))
print("Micro is " + str(micro))
enc_nn.train()
metric_nn.train()
return e_stop
amgnn.train()
return 100.0 * correct / total, loss_test_f
if __name__ == '__main__':
################################################################
print(time.strftime("%F"))
timedata = time.strftime("%F")
name = timedata + '-3-classe'
save_path = 'result\\{}'.format(name)
if name not in os.listdir('result\\'):
os.makedirs(save_path)
io = io_utils.IOStream(save_path + '\\run.log')
print('the result will be saved in :', save_path)
setup_seed(args.random_seed)
amgnn = models.create_models(args, cnn_dim1=2)
io.cprint(str(amgnn))
softmax_module = models.SoftmaxModule()
print(amgnn.parameters())
if args.cuda:
amgnn.cuda()
weight_decay = 0
opt_amgnn = optim.Adam(amgnn.parameters(),
lr=args.lr,
weight_decay=weight_decay)
