def run(data_train, data_test, kernel='rbf', C=1e-7, confusion=False): """Run the training script and yield accuracy, confusion :param data_train: Featurized training data, contains X and Y :param data_test: Featurized test data, contains X and Y :param kernel: kernel to apply during classification :param C: Weight of misclassification during classification :param confusion: Whether or not to print confusion matrix :return: None """ model = train(data_train, C=C, kernel=kernel) pred_train = predict(model, data_train) pred_test = predict(model, data_test) train_accuracy = accuracy_score(data_train['Y'], pred_train) test_accuracy = accuracy_score(data_test['Y'], pred_test) print('[%s][%.2E] \t Accuracy: \t (Train) %.4f \t (Val) %.4f' % (kernel, C, train_accuracy, test_accuracy)) if confusion: cnf_matrix = confusion_matrix(data_test['Y'], pred_test) plt.figure() plot_confusion_matrix(cnf_matrix, classes=CLASS_NAMES, title='Confusion matrix, without normalization') plt.savefig('../data/generated/cnf-%s.png' % str(time.time())[-6:]) # Plot normalized confusion matrix plt.figure() plot_confusion_matrix(cnf_matrix, classes=CLASS_NAMES, normalize=True, title='Normalized confusion matrix') plt.savefig('../data/generated/nm-cnf-%s.png' % str(time.time())[-6:]) return { 'model': model, 'train': train_accuracy, 'validation': test_accuracy }
def on_epoch_end(self, epoch, logs=None): batch_size = self.model.BATCH_SIZE class_color_map = self.model.CLS_COLOR_MAP # get first batch of dataset (lidar_input, lidar_mask), label, _ = self.dataset.take(1).get_single_element() probabilities, predictions = self.model([lidar_input, lidar_mask]) label = label[:self.num_images, :, :] predictions = predictions[:self.num_images, :, :].numpy() # label and prediction visualizations label_image = class_color_map[label.numpy().reshape(-1)].reshape( [self.num_images, label.shape[1], label.shape[2], 3]) pred_image = class_color_map[predictions.reshape(-1)].reshape( [self.num_images, label.shape[1], label.shape[2], 3]) # confusion matrix visualization figure = plot_confusion_matrix( self.model.miou_tracker.total_cm.numpy(), class_names=self.model.mc.CLASSES) cm_image = plot_to_image(figure) with self.custom_tb_writer.as_default(): tf.summary.image('Images/Depth Image', lidar_input.numpy()[:self.num_images, :, :, [4]], max_outputs=batch_size, step=epoch) tf.summary.image('Images/Label Image', label_image, max_outputs=batch_size, step=epoch) tf.summary.image('Images/Prediction Image', pred_image, max_outputs=batch_size, step=epoch) tf.summary.image("Confusion Matrix", cm_image, step=epoch) # Save IoU, Precision, Recall iou, recall, precision = confusion_matrix_to_iou_recall_precision( self.model.miou_tracker.total_cm) with self.custom_tb_writer.as_default(): for i, cls in enumerate(self.model.mc.CLASSES): tf.summary.scalar('IoU/' + cls, iou[i], step=epoch) tf.summary.scalar('Recall/' + cls, recall[i], step=epoch) tf.summary.scalar('Precision/' + cls, precision[i], step=epoch) super().on_epoch_end(epoch, logs)
def main(config, resume, config_testdata=None): #PREFERENCES outputOverlaycsv = False showHeatMap = False THRESHOLD = 0.45 TRUE_CLASS = 1 PRED_CLASS = 8 # set visualization preference print("GPUs available: " + str(torch.cuda.device_count())) os.environ["CUDA_VISIBLE_DEVICES"] = "0,1" device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') # setup data_loader instances data_loader = get_instance(module_data, 'data_loader_test', config_testdata) ''' data_loader = getattr(module_data, config['data_loader']['type'])( config['data_loader']['args']['data_dir'], batch_size=512, shuffle=False, validation_split=0.0, training=False, num_workers=2 ) ''' # build model architecture #config['arch']['args']['num_feature'] = 76 model = get_instance(module_arch, 'arch', config) print(model) if torch.cuda.is_available(): print("Using GPU: " + torch.cuda.get_device_name(0)) else: print("Using CPU to test") # get function handles of loss and metrics loss_fn = getattr(module_loss, config['loss']) criterion = loss_fn(None) # for imbalanced datasets #criterion = loss_fn(data_loader.dataset.weight.to(device)) # for imbalanced datasets metric_fns = [getattr(module_metric, met) for met in config['metrics']] # load state dict checkpoint = torch.load(resume) state_dict = checkpoint['state_dict'] if config['n_gpu'] > 1: model = torch.nn.DataParallel(model) model.load_state_dict(state_dict) # prepare model for testing device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') model = model.to(device) model.eval() total_loss = 0.0 total_metrics = torch.zeros(len(metric_fns)) #classes = ('endothelium', 'pct', 'vasculature') classes = ('s1', 's2s3', 'tal', 'dct', 'cd', 'cd45', 'nestin', 'cd31_glom', 'cd31_inter', 'cd45_1', 'cd45_2') all_pred = [] all_pred_k = [] all_true = [] all_softmax = [] thresh_true = [] thresh_pred = [] thresh_true_bad = [] thresh_pred_bad = [] below_thresh = 0 if showHeatMap: hm_layers = { 'final_layer': 'layer', 'fc_layer': 'fc_layer', 'conv_num': 17, 'fc_num': 3 } #need to set based on model heatmapper = classActivationMap.CAMgenerator(hm_layers, config, model) #heatmapper = classActivationMap.CAMgenerator3d(hm_layers, config, model) #for 3d data heatmapper.generateImage(num_images=10) with torch.no_grad(): for i, (data, target) in enumerate(tqdm(data_loader)): k = 2 data, target = data.to(device), target.to(device) #output = model(data) output = triple_prediction(data, model) #image = np.squeeze(data[0].cpu().data.numpy()) label = np.squeeze(target[0].cpu().data.numpy()) all_true.extend(target.cpu().data.numpy()) all_pred.extend(np.argmax(output.cpu().data.numpy(), axis=1)) mypred = torch.topk(output, k, dim=1)[1] all_pred_k.extend(mypred.cpu().data.numpy()) m = torch.nn.Softmax(dim=0) for i, row in enumerate(output.cpu()): sm = m(row) all_softmax.append(sm.data.numpy()) #print(np.max(sm.numpy(), axis=0)) if np.amax(sm.numpy(), axis=0) > THRESHOLD: thresh_true.append(target.cpu().data.numpy()[i]) thresh_pred.append(np.argmax(sm)) else: below_thresh += 1 thresh_true_bad.append(target.cpu().data.numpy()[i]) thresh_pred_bad.append(np.argmax(sm)) if i < 2: m = torch.nn.Softmax(dim=0) print("prediction percentages") print(m(output.cpu()[0])) print(all_true[i]) #all_softmax.extend(m(output.cpu())) #if i > 50: break # # save sample images, or do something with output here # # computing loss, metrics on test set loss = criterion(output, target) batch_size = data.shape[0] total_loss += loss.item() * batch_size for i, metric in enumerate(metric_fns): total_metrics[i] += metric(output.cpu(), target.cpu()) * batch_size correct = 0 all_pred_k = np.array(all_pred_k) all_true_k = np.array(all_true) #CREATE 8 CLASS SOFTMAX #very ugly please fix softmax_combined = np.zeros((len(all_softmax), len(all_softmax[0]) - 3)) for i, row in enumerate(all_softmax): max_pct = np.amax([row[0], row[1]]) #can use amax instead of sum max_cd45 = np.amax([row[5], row[9], row[10]]) #can use amax instead of sum myrow = row[1:len(all_softmax[0]) - 3 + 1] myrow[0] = max_pct myrow[4] = max_cd45 softmax_combined[i] = myrow all_pred_k_combined = torch.topk( torch.from_numpy(softmax_combined), k, dim=1)[1] + 1 all_pred_k = torch.from_numpy(all_pred_k) all_true_k = torch.from_numpy(np.array(all_true)) for i in range(k): correct += torch.sum(all_pred_k[:, i] == all_true_k).item() print("TOP K all classes = {}".format(correct / len(all_pred_k))) all_pred_k = np.array(all_pred_k) all_true_k = np.array(all_true) #APPLY THRESHOLD - in development softmax_above_threshold = np.amax(softmax_combined, axis=1) softmax_above_threshold_mask = softmax_above_threshold > THRESHOLD thresh_pred_v2 = np.array(all_pred)[softmax_above_threshold_mask] thresh_true_v2 = np.array(all_true)[softmax_above_threshold_mask] #REASSIGN LABELS all_results = [ all_pred, all_true, thresh_true, thresh_pred, thresh_true_bad, thresh_pred_bad, all_pred_k, all_true_k, thresh_pred_v2, thresh_true_v2 ] all_pred, all_true, thresh_true, thresh_pred, thresh_true_bad, thresh_pred_bad, all_pred_k, all_true_k, thresh_pred_v2, thresh_true_v2 = reassign_labels( all_results, 0, 1) all_pred, all_true, thresh_true, thresh_pred, thresh_true_bad, thresh_pred_bad, all_pred_k, all_true_k, thresh_pred_v2, thresh_true_v2 = reassign_labels( all_results, 9, 5) all_pred, all_true, thresh_true, thresh_pred, thresh_true_bad, thresh_pred_bad, all_pred_k, all_true_k, thresh_pred_v2, thresh_true_v2 = reassign_labels( all_results, 10, 5) #VIEW MISTAKE CLASS interogate_CM(all_pred, all_true, TRUE_CLASS, PRED_CLASS, data_loader, classes) correct = 0 all_pred_k = torch.from_numpy(all_pred_k) all_true_k = torch.from_numpy(np.array(all_true)) for i in range(k): correct += torch.sum(all_pred_k_combined[:, i] == all_true_k).item() print("TOP K Combined classes = {}".format(correct / len(all_pred_k_combined))) # SHOW PERCENTAGE OF IMAGES BELOW THRESHOLD AS CONFUSION MATRIX cm_all = confusion_matrix(all_true, all_pred) cm_below_thresh = confusion_matrix(thresh_true_bad, thresh_pred_bad) cm_above_thresh = confusion_matrix(thresh_true, thresh_pred) compare_CM(cm_above_thresh, cm_all, classes) if outputOverlaycsv: ids = data_loader.dataset.getIds() #softmax = pd.DataFrame(all_softmax) softmax = pd.DataFrame(softmax_combined) #ids = ids[:,1].reshape(ids.shape[0], 1) print(ids[0:5]) print(ids.shape) print(softmax.shape) frames = [ids, softmax, pd.DataFrame(all_true)] output_data = np.concatenate(frames, axis=1) print(output_data.shape) output_df = pd.DataFrame(output_data) output_df.to_csv('overlaycsv.csv', index=False, header=False) n_samples = len(data_loader.sampler) print("num test images = " + str(n_samples)) log = {'loss': total_loss / n_samples} log.update({ met.__name__: total_metrics[i].item() / n_samples for i, met in enumerate(metric_fns) }) for key in log: print("{} = {:.4f}".format(key, log[key])) #print(log) log['classes'] = classes log['test_targets'] = all_true log['test_predictions'] = all_pred print("My_metric is accuracy") print("Number of images below threshold: {}".format(below_thresh)) print("Number of images below threshold v2: {}".format( len(all_true) - len(thresh_pred_v2))) util.plot_confusion_matrix(all_true, all_pred, classes=classes, normalize=False) util.plot_confusion_matrix(thresh_true, thresh_pred, classes=classes, normalize=False) util.plot_confusion_matrix(thresh_true_bad, thresh_pred_bad, classes=classes, normalize=False)
def main(config, resume): device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') print("GPUs available: " + str(torch.cuda.device_count())) os.environ["CUDA_VISIBLE_DEVICES"] = "0,1" #data_loader = get_instance(module_data, 'data_loader', config) --> cant use this because it will auto split a validation set data_loader = hdf5_3d_dataloader( config['data_loader']['args']['hdf5_path'], batch_size=128, shuffle=False, validation_split=0.0, training=True, mean=config['data_loader']['args']['mean'], stdev=config['data_loader']['args']['stdev']) print(len(data_loader.dataset)) data_loader_test = get_instance(module_data, 'data_loader_test', config) # Load trained model, including fully connected model = get_instance(module_arch, 'arch', config) print(model) if torch.cuda.is_available(): print("Using GPU: " + torch.cuda.get_device_name(0)) else: print("Using CPU to test") loss_fn = getattr(module_loss, config['loss']) criterion = loss_fn(None) # load state dict checkpoint = torch.load(resume) state_dict = checkpoint['state_dict'] if config['n_gpu'] > 1: model = torch.nn.DataParallel(model) model.load_state_dict(state_dict) model = model.to(device) model.eval() classes = ('S1', 'PCT', 'TAL', 'DCT', 'CD', 'CD45', 'Nestin', 'CD31_glom', 'CD31_inter') #identify feature layer to separate save_output = SaveOutput() #print(model.module.conv_layer4) handle = model.module.fc6.register_forward_hook(save_output) #generate feature set all_true = [] feature_set = [] with torch.no_grad(): for i, (data, target) in enumerate(tqdm(data_loader)): data, target = data.to(device), target.to(device) output = model(data) all_true.extend(target.cpu().data.numpy()) print(i * 128) all_true = np.array(all_true) all_features = np.array(save_output.outputs) for item in all_features: feature_set.extend(item) all_features = np.array(feature_set) #all_features = np.reshape(all_features, (feature_shape[0]*feature_shape[1], feature_shape[2])) print(all_true.shape) print(all_features.shape) #create + train logistic model minmaxscaler = MinMaxScaler() all_features_scaled = minmaxscaler.fit_transform(all_features) clf = LogisticRegression(random_state=0, class_weight='balanced', verbose=1, solver='lbfgs', n_jobs=-1).fit(all_features_scaled, all_true) #solver options: saga, lbfgs #test logistic model save_output_test = SaveOutput() handle = model.module.fc6.register_forward_hook(save_output_test) all_true_test = [] feature_set_test = [] all_softmax_cnn = [] with torch.no_grad(): for i, (data, target) in enumerate(tqdm(data_loader_test)): data, target = data.to(device), target.to(device) output = model(data) all_true_test.extend(target.cpu().data.numpy()) #all_pred_cnn.extend(np.argmax(output.cpu().data.numpy(), axis=1)) m = torch.nn.Softmax(dim=0) for i, row in enumerate(output.cpu()): sm = m(row) all_softmax_cnn.append(sm.data.numpy()) all_true_test = np.array(all_true_test) all_features_test = np.array(save_output_test.outputs) for item in all_features_test: feature_set_test.extend(item) all_features_test = np.array(feature_set_test) print(all_true_test.shape) print(all_features_test.shape) all_pred = clf.predict(minmaxscaler.transform(all_features_test)) all_softmax_lr = clf.predict_proba(all_features_test) all_softmax_combined = all_softmax_lr + all_softmax_cnn all_pred_combined = np.argmax(all_softmax_combined, axis=1) #REASSIGN LABELS all_results = [all_pred, all_true_test, all_pred_combined] all_pred, all_true_test, all_pred_combined = reassign_labels( all_results, 0, 1) all_pred, all_true_test, all_pred_combined = reassign_labels( all_results, 9, 5) all_pred, all_true_test, all_pred_combined = reassign_labels( all_results, 10, 5) #display results util.plot_confusion_matrix(all_true_test, all_pred, classes=classes, normalize=False) #log regression util.plot_confusion_matrix( all_true_test, all_pred_combined, classes=classes, normalize=False) #average of log regression and cnn
def main(): parser = argparse.ArgumentParser( description="PyTorch DeeplabV3Plus Training") # cuda, seed and logging parser.add_argument('--gpu-ids', type=str, default='0', help='use which gpu to train, must be a \ comma-separated list of integers only (default=0)') # evaluation option parser.add_argument('--num_image', type=int, default=1, help='visualization number of image') parser.add_argument('--shuffle', type=bool, default=True, help='data list shuffle') parser.add_argument('--result', type=str, default='eachimage', choices=['eachimage', 'matrix'], help='data list shuffle') args = parser.parse_args() # default settings for epochs, batch_size and lr print(args) deeplab = Deeplab('cfg/deeplab.cfg') with open('./dataset/val.txt', 'r') as f: datalist = [] for data in f.readlines(): if '\n' in data: data = data[:-1] datalist.append(data) with open('./dataset/output/class_names.txt', 'r') as f: ClassName = [] for cl in f.readlines(): if '\n' in cl: cl = cl[:-1] ClassName.append(cl) if args.shuffle: random.shuffle(datalist) if args.result == 'matrix': start = time.time() for data in tqdm(datalist): src = Image.open('./dataset/output/JPEGImages/' + data + '.jpg').convert('RGB') tar = Image.open('./dataset/output/SegmentationClassPNG/' + data + '.png') deeplab.validation_matrix(src, data, tar) cm = deeplab.evaluator.confusion_matrix end = time.time() plot_confusion_matrix(cm / len(datalist), ClassName) plt.savefig('result/all_CM.png') print('Image : %d | Time : %.2f | fps : %.1f' % (len(datalist), end - start, len(datalist) / (end - start))) elif args.result == 'eachimage': for i in range(args.num_image): data = datalist[i] src = Image.open('./dataset/output/JPEGImages/' + data + '.jpg').convert('RGB') tar = Image.open('./dataset/output/SegmentationClassPNG/' + data + '.png') deeplab.validation(src, data, ClassName, tar)
def validation(self, src, data, ClassName, tar=None): self.model.eval() self.evaluator.reset() test_loss = 0.0 w, h = src.size sample = {'image': src, 'label': tar} sample = self.transform(sample) image, target = sample['image'], sample['label'] # for the dimension image = torch.unsqueeze(image, 0) target = torch.unsqueeze(target, 0) image, target = image.cuda(), target.cuda() with torch.no_grad(): output = self.model(image) imgs = image.data.cpu() lbl_pred = output.data.max(1)[1].cpu().numpy()[:, :, :] lbl_true = target.data.cpu() #cpu() for img, lt, lp in zip(imgs, lbl_true, lbl_pred): img, lt = self.untransform(img, lt) viz = fcn.utils.visualize_segmentation( lbl_pred=lp, lbl_true=None, img=img, n_class=10, label_names=[ ' ', '11_pforceps', '12_mbforceps', '13_mcscissors', '15_pcapplier', '18_pclip', '20_sxir', '19_mtclip', '17_mtcapplier', '14_graspers' ]) width = int(viz.shape[1] / 3 * 2) #Image.fromarray(viz[:,width:,:]).save(str(data)+'.jpg') def hide(plt): ax = plt.gca() ax.axes.xaxis.set_visible(False) ax.axes.yaxis.set_visible(False) plt.figure(figsize=(20, 10)) plt.subplot(2, 2, 1) plt.imshow(src) plt.title('Image') hide(plt) plt.subplot(2, 2, 2) plt.imshow(tar) plt.title('SegmentationClass') hide(plt) plt.subplot(2, 2, 3) plt.imshow( Image.open('./dataset/output/SegmentationClassVisualization/' + data + '.jpg')) plt.title('SegmentationVisuallization') hide(plt) plt.subplot(2, 2, 4) plt.imshow( cv2.resize(np.float32(Image.fromarray(viz[:, width:, :])) / 255, (w, h), interpolation=cv2.INTER_LINEAR)) plt.title('Prediction') hide(plt) if not os.path.isdir('result'): os.makedirs('result') plt.savefig('result/%s.png' % (data)) if tar is not None: loss = self.criterion(output, target) test_loss += loss.item() #tbar.set_description('Test loss: %.3f' % (test_loss)) pred = output.data.cpu().numpy() target = target.cpu().numpy() pred = np.argmax(pred, axis=1) # Add batch sample into evaluator self.evaluator.add_batch(target[:, 145:815, :], pred[:, 145:815, :]) # Fast test during the training Acc = self.evaluator.Pixel_Accuracy() Acc_class = self.evaluator.Pixel_Accuracy_Class() mIoU = self.evaluator.Mean_Intersection_over_Union() FWIoU = self.evaluator.Frequency_Weighted_Intersection_over_Union() print('Validation:') #print('[Epoch: %d, numImages: %5d]' % (epoch, self.args.batch_size + image.data.shape[0])) print("Acc:{}, Acc_class:{}, mIoU:{}, fwIoU: {}".format( Acc, Acc_class, mIoU, FWIoU)) print('Loss: %.3f' % test_loss) plot_confusion_matrix(self.evaluator.confusion_matrix, ClassName) plt.savefig('result/%s_CM.png' % (data))