def test_one_shot(args, model, test_samples=5000, partition='test'): io = io_utils.IOStream('checkpoints/' + args.exp_name + '/run.log') io.cprint('\n**** TESTING WITH %s ***' % (partition,)) loader = generator.Generator(args.dataset_root, args, partition=partition, dataset=args.dataset) [enc_nn, metric_nn, softmax_module] = model enc_nn.eval() metric_nn.eval() correct = 0 total = 0 iterations = int(test_samples/args.batch_size_test) for i in range(iterations): data = loader.get_task_batch(batch_size=args.batch_size_test, n_way=args.test_N_way, num_shots=args.test_N_shots, unlabeled_extra=args.unlabeled_extra) [x, labels_x_cpu, _, _, xi_s, labels_yi_cpu, oracles_yi, hidden_labels] = data if args.cuda: xi_s = [batch_xi.cuda() for batch_xi in xi_s] labels_yi = [label_yi.cuda() for label_yi in labels_yi_cpu] oracles_yi = [oracle_yi.cuda() for oracle_yi in oracles_yi] hidden_labels = hidden_labels.cuda() x = x.cuda() else: labels_yi = labels_yi_cpu xi_s = [Variable(batch_xi) for batch_xi in xi_s] labels_yi = [Variable(label_yi) for label_yi in labels_yi] oracles_yi = [Variable(oracle_yi) for oracle_yi in oracles_yi] hidden_labels = Variable(hidden_labels) x = Variable(x) # Compute embedding from x and xi_s z = enc_nn(x)[-1] zi_s = [enc_nn(batch_xi)[-1] for batch_xi in xi_s] # Compute metric from embeddings output, out_logits = metric_nn(inputs=[z, zi_s, labels_yi, oracles_yi, hidden_labels]) output = out_logits y_pred = softmax_module.forward(output) y_pred = y_pred.data.cpu().numpy() y_pred = np.argmax(y_pred, axis=1) labels_x_cpu = labels_x_cpu.numpy() labels_x_cpu = np.argmax(labels_x_cpu, axis=1) for row_i in range(y_pred.shape[0]): if y_pred[row_i] == labels_x_cpu[row_i]: correct += 1 total += 1 if (i+1) % 100 == 0: io.cprint('{} correct from {} \tAccuracy: {:.3f}%)'.format(correct, total, 100.0*correct/total)) io.cprint('{} correct from {} \tAccuracy: {:.3f}%)'.format(correct, total, 100.0*correct/total)) io.cprint('*** TEST FINISHED ***\n'.format(correct, total, 100.0 * correct / total)) enc_nn.train() metric_nn.train() return 100.0 * correct / total
help='Decreasing the learning rate every x iterations') args = parser.parse_args() def _init_(): if not os.path.exists('checkpoints'): os.makedirs('checkpoints') if not os.path.exists('checkpoints/'+args.exp_name): os.makedirs('checkpoints/'+args.exp_name) if not os.path.exists('checkpoints/'+args.exp_name+'/'+'models'): os.makedirs('checkpoints/'+args.exp_name+'/'+'models') os.system('cp main.py checkpoints'+'/'+args.exp_name+'/'+'main.py.backup') os.system('cp models/models.py checkpoints' + '/' + args.exp_name + '/' + 'models.py.backup') _init_() io = io_utils.IOStream('checkpoints/' + args.exp_name + '/run.log') io.cprint(str(args)) args.cuda = not args.no_cuda and torch.cuda.is_available() torch.manual_seed(args.seed) if args.cuda: io.cprint('Using GPU : ' + str(torch.cuda.current_device())+' from '+str(torch.cuda.device_count())+' devices') torch.cuda.manual_seed(args.seed) else: io.cprint('Using CPU') def train_batch(model, data): [enc_nn, metric_nn, softmax_module] = model [batch_x, label_x, batches_xi, labels_yi, oracles_yi, hidden_labels] = data
def test_one_shot(args, fold, test_root, model, test_samples=50, partition='test', io_path='run.log'): io = io_utils.IOStream(io_path) io.cprint('\n**** TESTING BEGIN ***') root = test_root loader = Generator(root, keys=['CN', 'MCI', 'AD']) [amgnn, softmax_module] = model amgnn.eval() correct = 0 total = 0 pre_all = [] pre_all_num = [] real_all = [] iterations = int(test_samples / args.batch_size_test) for i in range(iterations): data = loader.get_task_batch(batch_size=args.batch_size_test, n_way=args.test_N_way, num_shots=args.test_N_shots, unlabeled_extra=args.unlabeled_extra, cuda=args.cuda) [x_t, labels_x_cpu_t, _, _, xi_s, labels_yi_cpu, oracles_yi] = data z_clinical, z_mri_feature = x_t[:, 0, 0, 0:args. clinical_feature_num], x_t[:, :, :, args. clinical_feature_num:] zi_s_clinical = [ batch_xi[:, 0, 0, 0:args.clinical_feature_num] for batch_xi in xi_s ] zi_s_mri_feature = [ batch_xi[:, :, :, args.clinical_feature_num:] for batch_xi in xi_s ] adj = compute_adj(z_clinical, zi_s_clinical) x = x_t labels_x_cpu = labels_x_cpu_t if args.cuda: xi_s = [batch_xi.cuda() for batch_xi in zi_s_mri_feature] labels_yi = [label_yi.cuda() for label_yi in labels_yi_cpu] oracles_yi = [oracle_yi.cuda() for oracle_yi in oracles_yi] x = x.cuda() else: labels_yi = labels_yi_cpu xi_s = [Variable(batch_xi) for batch_xi in zi_s_mri_feature] labels_yi = [Variable(label_yi) for label_yi in labels_yi] oracles_yi = [Variable(oracle_yi) for oracle_yi in oracles_yi] z_mri_feature = Variable(z_mri_feature) # Compute metric from embeddings output, out_logits = amgnn(inputs=[ z_clinical, z_mri_feature, zi_s_clinical, xi_s, labels_yi, oracles_yi, adj ]) output = out_logits Y = softmax_module.forward(output) y_pred = softmax_module.forward(output) y_pred = y_pred.data.cpu().numpy() y_inter = [list(y_i) for y_i in y_pred] pre_all_num = pre_all_num + list(y_inter) y_pred = np.argmax(y_pred, axis=1) labels_x_cpu = labels_x_cpu.cpu().numpy() labels_x_cpu = np.argmax(labels_x_cpu, axis=1) pre_all = pre_all + list(y_pred) real_all = real_all + list(labels_x_cpu) for row_i in range(y_pred.shape[0]): if y_pred[row_i] == labels_x_cpu[row_i]: correct += 1 total += 1 labels_x_cpu = Variable(torch.cuda.LongTensor(labels_x_cpu)) loss_test = F.nll_loss(Y, labels_x_cpu) loss_test_f = float(loss_test) del loss_test io.cprint('real_label: ' + str(real_all)) io.cprint('pre_all: ' + str(pre_all)) io.cprint('pre_all_num: ' + str(pre_all_num)) io.cprint('{} correct from {} \tAccuracy: {:.3f}%)'.format( correct, total, 100.0 * correct / total)) io.cprint('*** TEST FINISHED ***\n'.format(correct, total, 100.0 * correct / total)) amgnn.train() return 100.0 * correct / total, loss_test_f
def test_one_shot(e_stop, test_cycle, args, model, test_samples=5000, partition='test'): """Function used to perform testing (validation and final test) Parameters: e_stop (early_stop): early stop which monitors when we stop training test_cycle (int): current iteration cycle - used when creating confusion matrix file names args (Namespace): arguments from argparse model (list) : contains cnn, gnn and softmax models test_samples (int) : number of samples to test partition (string) : train, val or test Returns: e_stop (early_stop) : we return the early_stop object which has been updated based upon the test """ io = io_utils.IOStream('checkpoints/' + args.exp_name + '/run.log') io.cprint('\n**** TESTING WITH %s ***' % (partition, )) loader = generator.Generator(args.dataset_root, args, partition=partition, dataset=args.dataset) [enc_nn, metric_nn, softmax_module] = model enc_nn.eval() metric_nn.eval() correct = 0 total = 0 iterations = int(test_samples / args.batch_size_test) true_list = [] predicted_list = [] with open( os.path.join('datasets', 'compacted_datasets', 'sensor_label_decoder.pickle'), 'rb') as handle: label_decoder = pickle.load(handle) sep = '\\' for temp in range(0, len(label_decoder)): label_decoder[temp] = label_decoder[temp].rsplit(sep, 1)[1] for i in range(iterations): data, labels_dict = loader.get_task_batch( batch_size=args.batch_size_test) [x, labels_x_cpu, _, x_global, xi_s, labels_yi_cpu] = data if args.cuda: xi_s = [batch_xi.cuda() for batch_xi in xi_s] labels_yi = [label_yi.cuda() for label_yi in labels_yi_cpu] x = x.cuda() else: labels_yi = labels_yi_cpu xi_s = [Variable(batch_xi) for batch_xi in xi_s] labels_yi = [Variable(label_yi) for label_yi in labels_yi] x = Variable(x) # Compute embedding from x and xi_s z = enc_nn(x) zi_s = [enc_nn(batch_xi) for batch_xi in xi_s] # Compute metric from embeddings output, out_logits = metric_nn(inputs=[z, zi_s, labels_yi]) output = out_logits y_pred = softmax_module.forward(output) y_pred = y_pred.data.cpu().numpy() y_pred = np.argmax(y_pred, axis=1) labels_x_cpu = labels_x_cpu.numpy() labels_x_cpu = np.argmax(labels_x_cpu, axis=1) for i in range(0, len(labels_x_cpu)): true_label = labels_dict[i, labels_x_cpu[i]] true_list.append(label_decoder[true_label]) predicted_label = labels_dict[i, y_pred[i]] predicted_list.append(label_decoder[predicted_label]) for row_i in range(y_pred.shape[0]): if y_pred[row_i] == labels_x_cpu[row_i]: correct += 1 total += 1 if (i + 1) % 100 == 0: io.cprint('{} correct from {} \tAccuracy: {:.3f}%)'.format( correct, total, 100.0 * correct / total)) acc = accuracy_score(true_list, predicted_list) micro = f1_score(true_list, predicted_list, average='weighted') macro = f1_score(true_list, predicted_list, average='macro') e_stop.update(micro, enc_nn, metric_nn) if partition == 'test' or (partition == 'val' and e_stop.improve): if partition == 'test': test_cycle = 999 #Print confusion matrix conf_mat.conf_mat(true_list, predicted_list, test_cycle, args.train, args.test) test_labels = sorted(set(true_list).union(set(predicted_list))) print( classification_report(true_list, predicted_list, target_names=test_labels)) print("Micro is " + str(micro)) enc_nn.train() metric_nn.train() return e_stop
amgnn.train() return 100.0 * correct / total, loss_test_f if __name__ == '__main__': ################################################################ print(time.strftime("%F")) timedata = time.strftime("%F") name = timedata + '-3-classe' save_path = 'result\\{}'.format(name) if name not in os.listdir('result\\'): os.makedirs(save_path) io = io_utils.IOStream(save_path + '\\run.log') print('the result will be saved in :', save_path) setup_seed(args.random_seed) amgnn = models.create_models(args, cnn_dim1=2) io.cprint(str(amgnn)) softmax_module = models.SoftmaxModule() print(amgnn.parameters()) if args.cuda: amgnn.cuda() weight_decay = 0 opt_amgnn = optim.Adam(amgnn.parameters(), lr=args.lr, weight_decay=weight_decay)