def validate(config, testloader, model, writer_dict): model.eval() ave_loss = AverageMeter() confusion_matrix = np.zeros( (config.DATASET.NUM_CLASSES, config.DATASET.NUM_CLASSES)) with torch.no_grad(): for _, batch in enumerate(testloader): image, label, _, _ = batch size = label.size() label = label.long().cuda() losses, pred = model(image, label) pred = F.upsample(input=pred, size=(size[-2], size[-1]), mode='bilinear') loss = losses.mean() ave_loss.update(loss.item()) confusion_matrix += get_confusion_matrix( label, pred, size, config.DATASET.NUM_CLASSES, config.TRAIN.IGNORE_LABEL) pos = confusion_matrix.sum(1) res = confusion_matrix.sum(0) tp = np.diag(confusion_matrix) IoU_array = (tp / np.maximum(1.0, pos + res - tp)) mean_IoU = IoU_array.mean() writer = writer_dict['writer'] global_steps = writer_dict['valid_global_steps'] writer.add_scalar('valid_loss', ave_loss.average(), global_steps) writer.add_scalar('valid_mIoU', mean_IoU, global_steps) writer_dict['valid_global_steps'] = global_steps + 1 return ave_loss.average(), mean_IoU, IoU_array
def validate(config, testloader, model, writer_dict): model.eval() ave_loss = AverageMeter() nums = config.MODEL.NUM_OUTPUTS confusion_matrix = np.zeros( (config.DATASET.NUM_CLASSES, config.DATASET.NUM_CLASSES, nums)) with torch.no_grad(): for idx, batch in enumerate(testloader): image, label, _, _ = batch size = label.size() image = image.cuda() label = label.long().cuda() losses, pred = model(image, label) if not isinstance(pred, (list, tuple)): pred = [pred] for i, x in enumerate(pred): x = F.interpolate( input=x, size=size[-2:], mode='bilinear', align_corners=config.MODEL.ALIGN_CORNERS ) confusion_matrix[..., i] += get_confusion_matrix( label, x, size, config.DATASET.NUM_CLASSES, config.TRAIN.IGNORE_LABEL ) if idx % 10 == 0: print(idx) loss = losses.mean() if dist.is_distributed(): reduced_loss = reduce_tensor(loss) else: reduced_loss = loss ave_loss.update(reduced_loss.item()) if dist.is_distributed(): confusion_matrix = torch.from_numpy(confusion_matrix).cuda() reduced_confusion_matrix = reduce_tensor(confusion_matrix) confusion_matrix = reduced_confusion_matrix.cpu().numpy() for i in range(nums): pos = confusion_matrix[..., i].sum(1) res = confusion_matrix[..., i].sum(0) tp = np.diag(confusion_matrix[..., i]) IoU_array = (tp / np.maximum(1.0, pos + res - tp)) mean_IoU = IoU_array.mean() if dist.get_rank() <= 0: logging.info('{} {} {}'.format(i, IoU_array, mean_IoU)) writer = writer_dict['writer'] global_steps = writer_dict['valid_global_steps'] writer.add_scalar('valid_loss', ave_loss.average(), global_steps) writer.add_scalar('valid_mIoU', mean_IoU, global_steps) writer_dict['valid_global_steps'] = global_steps + 1 return ave_loss.average(), mean_IoU, IoU_array
def testval(config, test_dataset, testloader, model, # sv_dir='', sv_pred=False): sv_dir='', sv_pred=True): model.eval() confusion_matrix = np.zeros( (config.DATASET.NUM_CLASSES, config.DATASET.NUM_CLASSES)) with torch.no_grad(): for index, batch in enumerate(tqdm(testloader)): image, label, _, name, *border_padding = batch size = label.size() pred = test_dataset.multi_scale_inference( config, model, image, scales=config.TEST.SCALE_LIST, flip=config.TEST.FLIP_TEST) if len(border_padding) > 0: border_padding = border_padding[0] pred = pred[:, :, 0:pred.size(2) - border_padding[0], 0:pred.size(3) - border_padding[1]] if pred.size()[-2] != size[-2] or pred.size()[-1] != size[-1]: pred = F.interpolate( pred, size[-2:], mode='bilinear', align_corners=config.MODEL.ALIGN_CORNERS ) confusion_matrix += get_confusion_matrix( label, pred, size, config.DATASET.NUM_CLASSES, config.TRAIN.IGNORE_LABEL) if sv_pred: sv_path = os.path.join(sv_dir, 'test_results_debug_1217') if not os.path.exists(sv_path): os.mkdir(sv_path) test_dataset.save_pred(pred, sv_path, name) if index % 100 == 0: logging.info('processing: %d images' % index) pos = confusion_matrix.sum(1) res = confusion_matrix.sum(0) tp = np.diag(confusion_matrix) IoU_array = (tp / np.maximum(1.0, pos + res - tp)) mean_IoU = IoU_array.mean() logging.info('mIoU: %.4f' % (mean_IoU)) pos = confusion_matrix.sum(1) res = confusion_matrix.sum(0) tp = np.diag(confusion_matrix) pixel_acc = tp.sum()/pos.sum() mean_acc = (tp/np.maximum(1.0, pos)).mean() IoU_array = (tp / np.maximum(1.0, pos + res - tp)) mean_IoU = IoU_array.mean() return mean_IoU, IoU_array, pixel_acc, mean_acc
def validate(config, testloader, model, writer_dict, device): rank = get_rank() world_size = get_world_size() model.eval() ave_loss = AverageMeter() confusion_matrix = np.zeros( (config.DATASET.NUM_CLASSES, config.DATASET.NUM_CLASSES)) confusion_matrix_sum = np.zeros( (config.DATASET.NUM_CLASSES, config.DATASET.NUM_CLASSES)) with torch.no_grad(): for _, batch in enumerate(testloader): image, label, boundary_gt, _, _ = batch size = label.size() image = image.to(device) boundary_gt = boundary_gt.to(device) label = label.long().to(device) losses, aux_loss, error_loss, losses_2, aux_loss_2, error_loss_2, preds = model( image, label, boundary_gt.float()) pred = F.upsample(input=preds[0], size=(size[-2], size[-1]), mode='bilinear') loss = (losses + 0.4 * aux_loss + 4 * error_loss + losses_2 + 0.4 * aux_loss_2 + 4 * error_loss_2).mean() reduced_loss = reduce_tensor(loss) ave_loss.update(reduced_loss.item()) confusion_matrix += get_confusion_matrix( label, pred, size, config.DATASET.NUM_CLASSES, config.TRAIN.IGNORE_LABEL) confusion_matrix = torch.from_numpy(confusion_matrix).to(device) reduced_confusion_matrix = reduce_tensor(confusion_matrix) confusion_matrix = reduced_confusion_matrix.cpu().numpy() pos = confusion_matrix.sum(1) res = confusion_matrix.sum(0) tp = np.diag(confusion_matrix) IoU_array = (tp / np.maximum(1.0, pos + res - tp)) mean_IoU = IoU_array.mean() print_loss = ave_loss.average() / world_size if rank == 0: writer = writer_dict['writer'] global_steps = writer_dict['valid_global_steps'] writer.add_scalar('valid_loss', print_loss, global_steps) writer.add_scalar('valid_mIoU', mean_IoU, global_steps) writer_dict['valid_global_steps'] = global_steps + 1 # cv2.imwrite(str(global_steps)+'_boundary.png', (preds[0][0][0].data.cpu().numpy()*255).astype(np.uint8)) # cv2.imwrite(str(global_steps) + '_error.png', (preds[2][0][0].data.cpu().numpy() * 255).astype(np.uint8)) cv2.imwrite( str(global_steps) + '_error.png', (preds[2][0][0].data.cpu().numpy() * 255).astype(np.uint8)) return print_loss, mean_IoU, IoU_array
def testval(config, test_dataset, testloader, model, sv_dir='', sv_pred=False): model.eval() confusion_matrix = np.zeros( (config.DATASET.NUM_CLASSES, config.DATASET.NUM_CLASSES)) with torch.no_grad(): for index, batch in enumerate(tqdm(testloader)): image, label, _, name = batch size = label.size() pred = test_dataset.multi_scale_inference( model, image, scales=config.TEST.SCALE_LIST, flip=config.TEST.FLIP_TEST) if pred.size()[-2] != size[-2] or pred.size()[-1] != size[-1]: pred = F.upsample(pred, (size[-2], size[-1]), mode='bilinear') confusion_matrix += get_confusion_matrix( label, pred, size, config.DATASET.NUM_CLASSES, config.TRAIN.IGNORE_LABEL) if sv_pred: sv_path = os.path.join(sv_dir,'test_val_results') if not os.path.exists(sv_path): os.mkdir(sv_path) test_dataset.save_pred(pred, sv_path, name) if index % 100 == 0: logging.info('processing: %d images' % index) pos = confusion_matrix.sum(1) res = confusion_matrix.sum(0) tp = np.diag(confusion_matrix) IoU_array = (tp / np.maximum(1.0, pos + res - tp)) mean_IoU = IoU_array.mean() logging.info('mIoU: %.4f' % (mean_IoU)) pos = confusion_matrix.sum(1) res = confusion_matrix.sum(0) tp = np.diag(confusion_matrix) pixel_acc = tp.sum()/pos.sum() mean_acc = (tp/np.maximum(1.0, pos)).mean() IoU_array = (tp / np.maximum(1.0, pos + res - tp)) mean_IoU = IoU_array.mean() return mean_IoU, IoU_array, pixel_acc, mean_acc
def log_confusion_matrix(epoch, logs): # Use the model to predict the values from the validation dataset. y_pred_raw = model.predict(test_batches) y_pred = np.argmax(y_pred_raw, axis=1) y_true = test_batches.classes # Get the confusion cm_image = get_confusion_matrix(y_true, y_pred, class_names=class_names, normalize='true') # Log the confusion matrix as an image summary. with file_writer_cm.as_default(): tf.summary.image("Confusion Matrix", cm_image, step=epoch)
def validate(config, testloader, model, writer_dict, device): rank = get_rank() world_size = get_world_size() model.eval() ave_loss = AverageMeter() confusion_matrix = np.zeros( (config.DATASET.NUM_CLASSES, config.DATASET.NUM_CLASSES)) with torch.no_grad(): for _, batch in enumerate(testloader): image, label, _, _ = batch size = label.size() image = image.to(device) label = label.long().to(device) losses, pred = model(image, label) pred = F.upsample(input=pred, size=( size[-2], size[-1]), mode='bilinear') loss = losses.mean() reduced_loss = reduce_tensor(loss) ave_loss.update(reduced_loss.item()) confusion_matrix += get_confusion_matrix( label, pred, size, config.DATASET.NUM_CLASSES, config.TRAIN.IGNORE_LABEL) confusion_matrix = torch.from_numpy(confusion_matrix).to(device) reduced_confusion_matrix = reduce_tensor(confusion_matrix) confusion_matrix = reduced_confusion_matrix.cpu().numpy() pos = confusion_matrix.sum(1) res = confusion_matrix.sum(0) tp = np.diag(confusion_matrix) IoU_array = (tp / np.maximum(1.0, pos + res - tp)) mean_IoU = IoU_array.mean() print_loss = ave_loss.average()/world_size if rank == 0: writer = writer_dict['writer'] global_steps = writer_dict['valid_global_steps'] writer.add_scalar('valid_loss', print_loss, global_steps) writer.add_scalar('valid_mIoU', mean_IoU, global_steps) writer_dict['valid_global_steps'] = global_steps + 1 return print_loss, mean_IoU, IoU_array
logits, dim=1) # (shape: (batch_size, num_classes, h, w)) p = p + p_value / M_float p_numpy = p.cpu().data.numpy().transpose( 0, 2, 3, 1) # (array of shape: (batch_size, h, w, num_classes)) seg_pred = np.argmax(p_numpy, axis=3).astype(np.uint8) m_seg_pred = np.ma.masked_array(seg_pred, mask=torch.eq(label, 255)) np.ma.set_fill_value(m_seg_pred, 20) seg_pred = m_seg_pred seg_gt = label.numpy().astype(np.int) ignore_index = seg_gt != 255 seg_gt = seg_gt[ignore_index] seg_pred = seg_pred[ignore_index] confusion_matrix += get_confusion_matrix(seg_gt, seg_pred, num_classes) entropy = -np.sum(p_numpy * np.log(p_numpy), axis=3) # (shape: (batch_size, h, w)) pred_label_imgs_raw = np.argmax(p_numpy, axis=3).astype(np.uint8) for i in range(image.size(0)): if i == 0: img = image[i].data.cpu().numpy() img = np.transpose(img, (1, 2, 0)) # (shape: (img_h, img_w, 3)) img = img + np.array([102.9801, 115.9465, 122.7717]) img = img[:, :, ::-1] cv2.imwrite(output_path + "/" + name[i] + "_img.png", img) label_img = label[i].data.cpu().numpy() label_img = label_img.astype(np.uint8)
def main(): args.time = get_currect_time() visualizer = Visualizer(args) log = Log(args) log.record_sys_param() log.record_file() """Set GPU Environment""" os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu trainloader = data.DataLoader(NYUDataset_crop_fast(args.data_list, args.random_scale, args.random_mirror, args.random_crop, args.batch_size, args.colorjitter),batch_size=args.batch_size, shuffle=True, num_workers=4, pin_memory=True) valloader = data.DataLoader(NYUDataset_val_full(args.data_val_list, args.random_scale, args.random_mirror, args.random_crop, 1), batch_size=8, shuffle=False, pin_memory=True) """Create Network""" deeplab = Res_Deeplab(num_classes=args.num_classes) print(deeplab) """Load pretrained Network""" saved_state_dict = torch.load(args.restore_from) print(args.restore_from) new_params = deeplab.state_dict().copy() for i in saved_state_dict: # Scale.layer5.conv2d_list.3.weight i_parts = i.split('.') # print i_parts # if not i_parts[1]=='layer5': if not i_parts[0] == 'fc': new_params['.'.join(i_parts[0:])] = saved_state_dict[i] deeplab.load_state_dict(new_params) model = deeplab model.cuda() model.train() model = model.float() model = DataParallelModel(model, device_ids=[0, 1]) criterion = CriterionDSN() criterion = DataParallelCriterion(criterion) optimizer = optim.SGD([{'params': filter(lambda p: p.requires_grad, model.parameters()), 'lr': args.learning_rate }], lr=args.learning_rate, momentum=args.momentum,weight_decay=args.weight_decay) optimizer.zero_grad() i_iter = 0 args.num_steps = len(trainloader) * args.epoch best_iou = 0.0 total = sum([param.nelement() for param in model.parameters()]) print(' + Number of params: %.2fM' % (total / 1e6)) for epoch in range(args.epoch): ## Train one epoch model.train() for batch in trainloader: start = timeit.default_timer() i_iter = i_iter + 1 images = batch['image'].cuda() labels = batch['seg'].cuda() HHAs = batch['HHA'].cuda() depths = batch['depth'].cuda() labels = torch.squeeze(labels,1).long() if (images.size(0) != args.batch_size): break optimizer.zero_grad() preds = model(images, HHAs, depths) loss = criterion(preds, labels) loss.backward() optimizer.step() if i_iter % 100 == 0: visualizer.add_scalar('learning_rate', args.learning_rate, i_iter) visualizer.add_scalar('loss', loss.data.cpu().numpy(), i_iter) current_lr = optimizer.param_groups[0]['lr'] end = timeit.default_timer() log.log_string( '====================> epoch=%03d/%d, iter=%05d/%05d, loss=%.3f, %.3fs/iter, %02d:%02d:%02d, lr=%.6f' % ( epoch, args.epoch, i_iter, len(trainloader)*args.epoch, loss.data.cpu().numpy(), (end - start), (int((end - start) * (args.num_steps - i_iter)) // 3600), (int((end - start) * (args.num_steps - i_iter)) % 3600 // 60), (int((end - start) * (args.num_steps - i_iter)) % 3600 % 60), current_lr)) if (epoch+1) % 40 == 0: adjust_learning_rate(optimizer, i_iter, args) if epoch % 5 == 0: model.eval() confusion_matrix = np.zeros((args.num_classes, args.num_classes)) loss_val = 0 log.log_string("====================> evaluating") for batch_val in valloader: images_val = batch_val['image'].cuda() labels_val = batch_val['seg'].cuda() labels_val = torch.squeeze(labels_val,1).long() HHAs_val = batch_val['HHA'].cuda() depths_val = batch_val['depth'].cuda() with torch.no_grad(): preds_val = model(images_val, HHAs_val, depths_val) loss_val += criterion(preds_val, labels_val) preds_val = torch.cat([preds_val[i][0] for i in range(len(preds_val))], 0) preds_val = F.upsample(input=preds_val, size=(480, 640), mode='bilinear', align_corners=True) preds_val = np.asarray(np.argmax(preds_val.cpu().numpy(), axis=1), dtype=np.uint8) labels_val = np.asarray(labels_val.cpu().numpy(), dtype=np.int) ignore_index = labels_val != 255 labels_val = labels_val[ignore_index] preds_val = preds_val[ignore_index] confusion_matrix += get_confusion_matrix(labels_val, preds_val, args.num_classes) loss_val = loss_val / len(valloader) pos = confusion_matrix.sum(1) res = confusion_matrix.sum(0) tp = np.diag(confusion_matrix) IU_array = (tp / np.maximum(1.0, pos + res - tp)) mean_IU = IU_array.mean() # getConfusionMatrixPlot(confusion_matrix) log.log_string('val loss' + ' ' + str(loss_val.cpu().numpy()) + ' ' + 'meanIU' + str(mean_IU) + 'IU_array' + str(IU_array)) visualizer.add_scalar('val loss', loss_val.cpu().numpy(), epoch) visualizer.add_scalar('meanIU', mean_IU, epoch) if mean_IU > best_iou: best_iou = mean_IU log.log_string('save best model ...') torch.save(deeplab.state_dict(), osp.join(args.snapshot_dir, 'model', args.dataset + NAME + 'best_iu' + '.pth')) if epoch % 5 == 0: log.log_string('save model ...') torch.save(deeplab.state_dict(),osp.join(args.snapshot_dir,'model', args.dataset+ NAME + str(epoch)+'.pth'))
def predict(self): ## Get image-label mapping image_label_dict = {} dataset_labels_file_path = 'datasets/densenet/ISIC2018_Task3_Training_GroundTruth.csv' with open(dataset_labels_file_path) as csvfile: read_csv = csv.reader(csvfile, delimiter=',') for index, row in enumerate(read_csv): ## Skip header if index == 0: continue label_one_hot_encoding = [ int(round(float(row[i + 1]), 0)) for i in range(7) ] image_label_dict[row[0]] = np.argmax(label_one_hot_encoding) ## Get image paths image_paths = utils_image.get_images_path_list_from_dir( self.config.tfrecords_path_train, img_format='jpg') # image_paths = utils_image.get_images_path_list_from_dir(self.config.tfrecords_path_test, img_format='jpg') # image_paths = utils_image.get_images_path_list_from_dir(self.config.tfrecords_path_val, img_format='jpg') ## Sample n images random.shuffle(image_paths) image_paths = image_paths[0:self.config.predict_num_images] ## Get gt_labels gt_labels = [] for image_path in image_paths: image_name = os.path.basename(image_path).rsplit('.', 1)[0] gt_labels.append(image_label_dict[image_name]) images = [] for image_path in image_paths: ## Load image image = Image.open(image_path) ## Resize and center crop image. size: (width, height) image = ImageOps.fit( image, (self.config.tfr_image_width, self.config.tfr_image_height), Image.LANCZOS, 0, (0.5, 0.5)) # img = cv2.imread(image_paths[i]) # img = cv2.resize(img, (224, 224)) ## Preprocess images image = np.float32(np.array(image)) image = self.data.preprocess_data(image) images.append(image) images = np.array(images) logging.debug('model_name {}'.format(self.config.model_name)) logging.debug('images {}'.format(images.shape)) # TODO: Don't shuffle else labels will mismatch x_key = self.config.model_name + '_input' predict_input_fn = tf.estimator.inputs.numpy_input_fn( x={x_key: images}, y=None, batch_size=self.config.batch_size_pred, num_epochs=1, shuffle=False, queue_capacity=1000, # In order to have predicted and repeatable order of reading and enqueueing, # such as in prediction and evaluation mode, num_threads should be 1. num_threads=1) # with tf.Session() as sess: # init_op = tf.global_variables_initializer() # sess.run(init_op) # # sess.run(self.model.model.output, feed_dict={self.model.model.input:images}) # # sess.run(self.model.model['class_prob'], feed_dict={self.model.model.input:images}) # # op = self.model.model.get_layer('class_prob') # # op = self.model.model.get_layer('class_prob') # # op = self.model.model.layers[0] # op = self.model.model.get_layer('dense_1') # out = sess.run(op, # feed_dict={self.model.model.input:images}) # logging.debug('out {}'.format(out)) # exit(0) checkpoint_path = None if not self.config.predict_weights_path: checkpoint_path = self.config.predict_weights_path # NOTE: predictions is <generator object Estimator.predict> and hence (maybe) we can dereference it only once. # TODO: Check predict_keys predictions = self.estimator.predict(input_fn=predict_input_fn, checkpoint_path=checkpoint_path, hooks=None, predict_keys=None, yield_single_examples=True) class_prob = [p['class_prob'] for p in predictions] # class_prob = [p['dense_4'] for p in predictions] pred_labels = np.argmax(np.array(class_prob), axis=1) for gt_label, pred_label in zip(gt_labels, pred_labels): print('GT, PRED: [{}, {}]'.format(gt_label, pred_label)) ## Confusion matrix # https://stackoverflow.com/questions/41617463/tensorflow-confusion-matrix-in-tensorboard confusion = tf.confusion_matrix(labels=gt_labels, predictions=pred_labels, num_classes=self.config.num_classes) logging.debug('Row(GT), Col(Pred)') with tf.Session() as sess: print(sess.run(confusion)) # Plot and save confusion matrix utils.get_confusion_matrix(self.config, gt_labels, pred_labels)
def predict_cascade(self, mode_ds='test_ds'): # TODO: Hardcoding labels_binary = {"REST": 0, "NV": 1} labels_multi6 = { "MEL": 0, "NV": 1, "BCC": 2, "AKIEC": 3, "BKL": 4, "DF": 5, "VASC": 6 } labels_binary_inv = {v: k for k, v in labels_binary.items()} labels_multi6_inv = {v: k for k, v in labels_multi6.items()} print('aaa') self.config.exp_cascade_name = 'binary' pkl_path = os.path.join(self.config.output_path, self.config.exp_name, self.config.exp_cascade_name, 'pkl') with open(os.path.join(pkl_path, 'labels_gt.pkl'), 'rb') as handle: labels_gt_binary = pickle.load(handle) with open(os.path.join(pkl_path, 'labels_pred_prob.pkl'), 'rb') as handle: labels_pred_prob_binary = pickle.load(handle) with open(os.path.join(pkl_path, 'labels_pred_cls.pkl'), 'rb') as handle: labels_pred_cls_binary = pickle.load(handle) self.config.exp_cascade_name = 'multi6' pkl_path = os.path.join(self.config.output_path, self.config.exp_name, self.config.exp_cascade_name, 'pkl') with open(os.path.join(pkl_path, 'labels_gt.pkl'), 'rb') as handle: labels_gt_multi6 = pickle.load(handle) with open(os.path.join(pkl_path, 'labels_pred_prob.pkl'), 'rb') as handle: labels_pred_prob_multi6 = pickle.load(handle) with open(os.path.join(pkl_path, 'labels_pred_cls.pkl'), 'rb') as handle: labels_pred_cls_multi6 = pickle.load(handle) labels_pred_prob = [] labels_pred_cls = [] labels_gt = labels_gt_binary # for lgb, lpb, lcb, lgm, lpm, lcm in zip(labels_gt_binary, labels_pred_prob_binary, labels_pred_cls_binary, labels_gt_multi6, labels_pred_prob_multi6, labels_pred_cls_multi6): for i in range(len(labels_gt_binary)): # binary #if (labels_pred_cls_binary[i] == labels_binary['NV']) and (np.max(labels_pred_prob_binary[i]) > 0.5): if (labels_pred_cls_binary[i] == labels_binary['NV']): labels_pred_cls.append(labels_pred_cls_binary[i]) prob = 7 * [ labels_pred_prob_binary[i][labels_binary['REST']] / 6.0 ] prob[labels_binary['NV']] = labels_pred_prob_binary[i][ labels_binary['NV']] labels_pred_prob.append(prob) #labels_pred_prob.append(np.max(labels_pred_prob_binary[i])) # multi6 else: labels_pred_cls.append(labels_pred_cls_multi6[i]) labels_pred_prob.append(labels_pred_prob_multi6[i]) ## Plot ROC curve #utils.gen_roc_curve(self.config, labels_gt, labels_pred_prob, mode_ds) ## Plot PR Curve utils.gen_precision_recall_curve(self.config, labels_pred_cls, labels_pred_prob, mode_ds) ## Confusion matrix # confusion = tf.confusion_matrix(labels=labels_gt, predictions=labels_pred_cls, num_classes=self.config.num_classes) # logging.debug('Row(GT), Col(Pred)') # with tf.Session() as sess: # print(sess.run(confusion)) ## Plot and save confusion matrix utils.get_confusion_matrix(self.config, labels_gt, labels_pred_cls, mode_ds) ## Print PR and F1 utils.summary_pr_fscore(self.config, labels_gt, labels_pred_cls, self.config.labels) ## Plot Metrics utils.get_metrics(self.config, labels_gt, labels_pred_cls, mode_ds)
def predict(self, mode_ds='test_ds', mixed=False): if mode_ds == 'train_ds': image_paths = utils_image.get_images_path_list_from_dir( self.config.tfrecords_path_train, img_format='jpg') elif mode_ds == 'val_ds': images_path = utils_image.get_images_path_list_from_dir( self.config.tfrecords_path_val, img_format='jpg') elif mode_ds == 'test_ds': images_path = utils_image.get_images_path_list_from_dir( self.config.tfrecords_path_test, img_format='jpg') else: logging.error('Unknown mode_ds {}', mode_ds) exit(1) ## Get image-label mapping image_label_dict = {} dataset_labels_file_path = 'datasets/densenet/ISIC2018_Task3_Training_GroundTruth.csv' with open(dataset_labels_file_path) as csvfile: read_csv = csv.reader(csvfile, delimiter=',') for index, row in enumerate(read_csv): ## Skip header if index == 0: continue label_one_hot_encoding = [ int(round(float(row[i + 1]), 0)) for i in range(7) ] image_label_dict[row[0]] = np.argmax(label_one_hot_encoding) ## Sample n images # random.shuffle(images_path) images_path = images_path[0:self.config.predict_num_images] ## Get labels_gt labels_gt = [] for image_path in images_path: # TODO: Image name should have no dot # image_name = os.path.basename(image_path).split('.', 1)[0] image_name = os.path.basename(image_path).rsplit('.', 1)[0] labels_gt.append(image_label_dict[image_name]) images_path = images_path[0:self.config.predict_num_images] images = [] for image_path in images_path: ## Load image image = Image.open(image_path) ## Resize and center crop image. size: (width, height) image = ImageOps.fit( image, (self.config.tfr_image_width, self.config.tfr_image_height), Image.LANCZOS, 0, (0.5, 0.5)) ## Preprocess images image = np.float32(np.array(image)) image = self.data.preprocess_data(image) images.append(image) features = np.array(images) logging.debug('model_name {}'.format(self.config.model_name)) logging.debug('features {}'.format(features.shape)) ## Predict in batches num_images = len(images_path) batch_size = self.config.batch_size_pred iters = int(num_images / batch_size) logging.debug('num_images {}'.format(num_images)) logging.debug('batch_size {}'.format(batch_size)) labels_pred_cls = [] labels_pred_prob = [] idx_start = 0 idx_end = 0 for iter_no in range(iters): idx_start = iter_no * batch_size idx_end = idx_start + batch_size logging.debug('idx:[{}-{}]'.format(idx_start, idx_end)) labels_pred_prob_batch, labels_pred_cls_batch = self.sess.run( [ self.model.labels_pred_prob, self.model.labels_pred_cls, ], feed_dict={ self.model.features: features[idx_start:idx_end], }) logging.debug('labels_gt {}'.format( np.array(labels_gt[idx_start:idx_end]))) logging.debug( 'labels_pred_cls_batch {}'.format(labels_pred_cls_batch)) # logging.debug('labels_pred_prob_batch {}'.format(labels_pred_prob_batch)) labels_pred_cls = labels_pred_cls + labels_pred_cls_batch.tolist() labels_pred_prob = labels_pred_prob + labels_pred_prob_batch.tolist( ) ## For images < batch_size and For images which do not fit the last batch idx_start = iters * batch_size idx_end = idx_start + (num_images % batch_size) logging.debug('idx:[{}-{}]'.format(idx_start, idx_end)) if (num_images % batch_size): labels_pred_prob_batch, labels_pred_cls_batch = self.sess.run( [ self.model.labels_pred_prob, self.model.labels_pred_cls, ], feed_dict={ self.model.features: features[idx_start:idx_end], }) logging.debug('labels_gt {}'.format( labels_gt[idx_start:idx_end])) logging.debug( 'labels_pred_cls_batch {}'.format(labels_pred_cls_batch)) # logging.debug('labels_pred_prob_batch {}'.format(labels_pred_prob_batch)) labels_pred_cls = labels_pred_cls + labels_pred_cls_batch.tolist() labels_pred_prob = labels_pred_prob + labels_pred_prob_batch.tolist( ) for label_gt, label_pred_cls in zip(labels_gt, labels_pred_cls): print('GT, PRED: [{}, {}]'.format(label_gt, label_pred_cls)) pkl_path = os.path.join(self.config.output_path, self.config.exp_name, self.config.exp_cascade_name, 'pkl') os.makedirs(pkl_path, exist_ok=True) with open(os.path.join(pkl_path, 'labels_gt.pkl'), 'wb') as handle: pickle.dump(labels_gt, handle, protocol=pickle.HIGHEST_PROTOCOL) with open(os.path.join(pkl_path, 'labels_pred_prob.pkl'), 'wb') as handle: pickle.dump(labels_pred_prob, handle, protocol=pickle.HIGHEST_PROTOCOL) with open(os.path.join(pkl_path, 'labels_pred_cls.pkl'), 'wb') as handle: pickle.dump(labels_pred_cls, handle, protocol=pickle.HIGHEST_PROTOCOL) ### ANALYSIS ### ## Plot ROC curve utils.gen_roc_curve(self.config, labels_gt, labels_pred_prob, mode_ds) ## Plot PR Curve utils.gen_precision_recall_curve(self.config, labels_pred_cls, labels_pred_prob, mode_ds) ## Confusion matrix # confusion = tf.confusion_matrix(labels=labels_gt, predictions=labels_pred_cls, num_classes=self.config.num_classes) # logging.debug('Row(GT), Col(Pred)') # with tf.Session() as sess: # print(sess.run(confusion)) ## Plot and save confusion matrix utils.get_confusion_matrix(self.config, labels_gt, labels_pred_cls, mode_ds) ## Print PR and F1 utils.summary_pr_fscore(self.config, labels_gt, labels_pred_cls, self.config.labels) ## Plot Metrics utils.get_metrics(self.config, labels_gt, labels_pred_cls, mode_ds)
def testval(config, test_dataset, testloader, model, sv_dir='', sv_pred=False): model.eval() confusion_matrix = np.zeros( (config.DATASET.NUM_CLASSES, config.DATASET.NUM_CLASSES)) with torch.no_grad(): for index, batch in enumerate(tqdm(testloader)): image, label, _, name, *border_padding = batch size = label.size() pred = test_dataset.multi_scale_inference( config, model, image, scales=config.TEST.SCALE_LIST, flip=config.TEST.FLIP_TEST) if len(border_padding) > 0: border_padding = border_padding[0] pred = pred[:, :, 0:pred.size(2) - border_padding[0], 0:pred.size(3) - border_padding[1]] if pred.size()[-2] != size[-2] or pred.size()[-1] != size[-1]: pred = F.interpolate(pred, size[-2:], mode='nearest') # # crf used for post-processing # postprocessor = DenseCRF( ) # # image # mean=[0.485, 0.456, 0.406], # std=[0.229, 0.224, 0.225] # timage = image.squeeze(0) # timage = timage.numpy().copy().transpose((1,2,0)) # timage *= std # timage += mean # timage *= 255.0 # timage = timage.astype(np.uint8) # # pred # tprob = torch.softmax(pred, dim=1)[0].cpu().numpy() # pred = postprocessor(np.array(timage, dtype=np.uint8), tprob) # pred = torch.from_numpy(pred).unsqueeze(0) confusion_matrix += get_confusion_matrix( label, pred, size, config.DATASET.NUM_CLASSES, config.TRAIN.IGNORE_LABEL) if sv_pred: sv_path = os.path.join(sv_dir, 'test_results') if not os.path.exists(sv_path): os.mkdir(sv_path) test_dataset.save_pred2(image, pred, sv_path, name) if index % 100 == 0: logging.info('processing: %d images' % index) pos = confusion_matrix.sum(1) res = confusion_matrix.sum(0) tp = np.diag(confusion_matrix) IoU_array = (tp / np.maximum(1.0, pos + res - tp)) mean_IoU = IoU_array.mean() logging.info('mIoU: %.4f' % (mean_IoU)) pos = confusion_matrix.sum(1) res = confusion_matrix.sum(0) tp = np.diag(confusion_matrix) pixel_acc = tp.sum() / pos.sum() mean_acc = (tp / np.maximum(1.0, pos)).mean() IoU_array = (tp / np.maximum(1.0, pos + res - tp)) mean_IoU = IoU_array.mean() return mean_IoU, IoU_array, pixel_acc, mean_acc
pallete="Set2", title="Leave-One-Subject-Out Cross Validation", showmeans=True, width=800, height=500) st.plotly_chart(fig) """ ### F-measure """ fig = pu.plotly_multi_boxplot(f_measure, x="Target", y="F-measure", hue="Model", pallet="Set2", title="F-measure per Target", showmeans=True, width=1000, height=500) st.plotly_chart(fig) """ ### Confusion Matrix """ model_select = st.selectbox("Model", accuracy["Model"].unique()) conf_mat = utils.get_confusion_matrix(model_select) fig = pu.plotly_heatmap(conf_mat) st.plotly_chart(fig) if st.checkbox("Show Confusion Matrix via DataFrame"): st.write(conf_mat)
def testval(num_class, ignore_label, rowSize, test_dataset, testloader, model, sv_dir='', sv_pred=False): model.eval() confusion_matrix = np.zeros((num_class + 1, num_class + 1)) with torch.no_grad(): for index, batch in enumerate(tqdm(testloader)): image, label, _, name = batch size = label.size() pred = torch.rand(size[1], size[2], num_class) for j in range(0, size[1], rowSize): # ¶àÐвâÊÔ row_size = rowSize if size[1] - j < rowSize: row_size = size[1] - j imageCubes, labelCubes = createTestCube(image[0], label[0], windowSize=11, r=j, size=row_size) hsiDataset = datasets.hsicube(imageCubes, labelCubes) hsiDataloader = torch.utils.data.DataLoader( hsiDataset, batch_size=size[2] * row_size) for _, (inputCube, _) in enumerate(hsiDataloader): inputCube = inputCube.cuda() with torch.no_grad(): output = model.forward(inputCube) pred[j:j + row_size, :, :] = output.reshape( (row_size, size[2], num_class)) print('\r' + '{}'.format(j), end='', flush=True) confusion_matrix += get_confusion_matrix(label, pred, size, num_class + 1, ignore_label) if sv_pred: sv_path = os.path.join(sv_dir, 'test_val_results') if not os.path.exists(sv_path): os.mkdir(sv_path) test_dataset.save_pred(pred, sv_path, name) # test_dataset.save_pred_gray(pred, sv_path, name) # if index % 100 == 0: logging.info('processing: %d images' % index) pos = confusion_matrix.sum(1) res = confusion_matrix.sum(0) tp = np.diag(confusion_matrix) IoU_array = (tp / np.maximum(1.0, pos + res - tp)) mean_IoU = IoU_array.mean() logging.info('mIoU: %.4f' % (mean_IoU)) pos = confusion_matrix.sum(1) res = confusion_matrix.sum(0) tp = np.diag(confusion_matrix) pixel_acc = tp.sum() / pos.sum() mean_acc = (tp / np.maximum(1.0, pos)).mean() IoU_array = (tp / np.maximum(1.0, pos + res - tp)) mean_IoU = IoU_array.mean() return mean_IoU, IoU_array, pixel_acc, mean_acc