def test_model(model, loader, is_nn): print(f"Testing {type(model).__name__}") if is_nn: preds_list = [] for batch in loader.get_batch(val=False): msgs = LongTensor(batch[1]) msg_len_ordered, msg_perm = (msgs != model.pad).sum(dim=1).sort(descending=True) # yapf: disable _, undo_msg_perm = msg_perm.sort() msgs = msgs[msg_perm] with torch.no_grad(): model.zero_grad() preds = model.forward(msgs, msg_len_ordered, is_training=False) preds = preds[undo_msg_perm] preds_list.append(preds.cpu()) preds = torch.max(torch.cat(preds_list, 0), 1)[1] preds = preds[:len(loader.data)] + 1 else: preds = model.test(loader.data) if model.argv.output is None: out_path = os.path.join(model.argv.res_root, model.argv.model_folder, f'test_epoch{model.argv.checkpoint_ver}.txt') else: out_path = model.argv.output utils.save_preds(out_path, preds)
def evaluate(net, criterion, experiment_dir, args, valid_loader, plot_name): net.eval() epoch_loss = 0 nb_batches = len(valid_loader) nb_eval = nb_batches * args.batch_size # Track samples by batches for scoring pred_y = np.zeros((nb_eval)) true_y = np.zeros((nb_eval)) weights = np.zeros((nb_eval)) evt_id = [] f_name = [] logging.info("Evaluating {} {} samples.".format(nb_eval,plot_name)) with torch.autograd.no_grad(): for i, batch in enumerate(valid_loader): X, y, w, adj_mask, batch_nb_nodes, evt_ids, evt_names = batch out = net(X, adj_mask, batch_nb_nodes) loss = criterion(out, y, w) epoch_loss += loss.item() # Track predictions, truth, weights over batches beg = i * args.batch_size end = (i+1) * args.batch_size pred_y[beg:end] = out.data.cpu().numpy() true_y[beg:end] = y.data.cpu().numpy() weights[beg:end] = w.data.cpu().numpy() if plot_name==TEST_NAME: evt_id.extend(evt_ids) f_name.extend(evt_names) # Print running loss 2 times if (((i+1) % (nb_batches//2)) == 0): nb_proc = (i+1)*args.batch_size logging.info(" {:5d}: {:.9f}".format(nb_proc, epoch_loss/nb_proc)) # Score predictions, save plots, and log performance epoch_loss /= nb_eval # Normalize loss tpr, roc = utils.score_plot_preds(true_y, pred_y, weights, experiment_dir, plot_name, args.eval_tpr) logging.info("{}: loss {:>.3E} -- AUC {:>.3E} -- TPR {:>.3e}".format( plot_name, epoch_loss, roc, tpr)) if plot_name == TEST_NAME: utils.save_test_scores(nb_eval, epoch_loss, tpr, roc, experiment_dir) utils.save_preds(evt_id, f_name, pred_y, experiment_dir) return (tpr, roc, epoch_loss)
# Generators training_generator = DataGenerator(training_ids, labels, **params) params['batch_size'] = 128 validation_generator = DataGenerator(validation_ids, labels, **params) # Design model model = get_model(params, True) print(model.summary()) # Train checkpointer = ModelCheckpoint(filepath='model.h5', verbose=2, save_best_only=True, save_weights_only=False) model.fit_generator(generator=training_generator, validation_data=validation_generator, callbacks=[checkpointer], epochs=1) # Predict params['shuffle'] = False params['augment'] = False params['batch_size'] = 1 params['dir_path'] = '/tmp/human_atlas/testing_data/' testing_generator = DataGenerator(testing_ids, labels=None, **params) preds = model.predict_generator(testing_generator, steps=len(testing_ids), verbose=1) save_numpy(preds) save_preds(preds, testing_ids) print(preds, preds.shape, len(testing_ids))
acc = accuracy_score(val_gts, val_preds) i += 1 print(f"[{i}/{len(file_list)/2}]acc: {acc*100:.2f} {file_path}") else: with open(file_path, 'r') as f: test_pred_all.append([int(line) for line in f]) val_pred_all = np.array(val_pred_all) val_gts = np.array(val_gts) test_pred_all = np.array(test_pred_all) assert val_pred_all.shape[1] == val_gts.shape[0] assert val_pred_all.shape[0] == test_pred_all.shape[0] sys.stdout.flush() val_pred_max_freq, val_pred_max_cnt = scipy.stats.mode(val_pred_all) target_names = [str(l) for l in range(1, 21)] acc = accuracy_score(val_gts, val_pred_max_freq.T) print(acc) print( classification_report(val_gts, val_pred_max_freq.T, target_names=target_names)) out_path = os.path.join(res_root, f'ensemble_val_acc{acc*100:0.2f}.png') utils.plot_confusion_matrix(target_names, val_gts, val_pred_max_freq.T, out_path) test_pred_max_freq, test_pred_max_cnt = scipy.stats.mode(test_pred_all) test_out_path = os.path.join(res_root, f'ensemble_test_acc{acc*100:0.2f}.txt') utils.save_preds(test_out_path, test_pred_max_freq.squeeze()) print()
def runModel(datagen, model, optimizer, class_wts, process, batch_size, n_batches, loss_wts): ''' process : 'trn', 'val' or 'tst' ''' running_loss = 0 pred_list = [] label_list = [] soft_pred_list = [] all_file_list = [] with trange(n_batches, desc=process, ncols=100) as t: for m in range(n_batches): data, labels, filenames = datagen.__next__() labels_one_hot = utils.get_one_hot(labels).cuda() if process == 'trn': optimizer.zero_grad() model.train() pred, aux_pred = model.forward(data) pred = F.softmax(pred, 1) aux_pred = F.softmax(aux_pred, 1) loss = 0 for i in range(2): loss += loss_wts[0] * utils.weightedBCE(class_wts[i], pred[:, i], (labels_one_hot [:, i]))\ + loss_wts[1] * utils.weightedBCE(class_wts[i], aux_pred[:, i], (labels_one_hot [:, i])) loss.backward() if torch.isnan(loss): pdb.set_trace() optimizer.step() elif process == 'val' or process == 'tst': model.eval() with torch.no_grad(): pred = F.softmax(model.forward(data), 1) loss = utils.weightedBCE(class_wts[0], pred[:, 0], labels_one_hot[:, 0])\ + utils.weightedBCE(class_wts[1], pred[:, 1], labels_one_hot[:, 1]) running_loss += loss hard_pred = torch.argmax(pred, 1) pred_list.append(hard_pred.cpu()) soft_pred_list.append(pred.detach().cpu()) label_list.append(labels.cpu()) all_file_list += filenames t.set_postfix(loss=running_loss.item() / (float(m + 1) * batch_size)) t.update() finalLoss = running_loss / (float(m + 1) * batch_size) # if process != 'trn': # pred_list, soft_pred_list, label_list = utils.test_time_aug( # all_file_list, # soft_pred_list, # label_list, 3) acc = utils.globalAcc(pred_list, label_list) if not isinstance(pred_list, torch.Tensor): f1 = sklearn.metrics.f1_score(torch.cat(label_list), torch.cat(pred_list), labels=None) else: f1 = sklearn.metrics.f1_score(label_list, pred_list, labels=None) auroc, auprc, fpr_tpr_arr, precision_recall_arr = utils.AUC( soft_pred_list, label_list) metrics = Metrics(finalLoss, acc, f1, auroc, auprc, fpr_tpr_arr, precision_recall_arr) utils.save_preds(soft_pred_list, pred_list, label_list, all_file_list, args.savename, process) return metrics
# -*- coding: utf-8 -*- """ Created on Tue Nov 6 20:53:36 2018 @author: Arpit """ from utils import save_preds, get_songs, split_data, to_label from data_processing import get_image_data from model import Model # get names of all the songs songs = get_songs() # split them in training and valid sets according to given percentage songs_train, songs_valid = split_data(songs, 0.85) # get actual spectrogram(2d np.array) data for the songs X_train, Y_train = get_image_data('train', songs_train) X_valid, Y_valid = get_image_data('valid', songs_valid) # get names and spectrogram data for the final testing set(which is to be uploaded) X_test, keys = get_image_data('test') model = Model(False) model.train(X_train, Y_train, X_valid, Y_valid, 5000) preds = model.predict(X_test) preds = [to_label(pred) for pred in preds] save_preds(keys, preds, 'predictions.csv')