def evaluate(model=None, inp_images=None, annotations=None, inp_images_dir=None, annotations_dir=None, checkpoints_path=None): if model is None: assert ( checkpoints_path is not None), "Please provide the model or the checkpoints_path" model = model_from_checkpoint_path(checkpoints_path) if inp_images is None: assert (inp_images_dir is not None), "Please privide inp_images or inp_images_dir" assert (annotations_dir is not None), "Please privide inp_images or inp_images_dir" paths = get_pairs_from_paths(inp_images_dir, annotations_dir) paths = list(zip(*paths)) inp_images = list(paths[0]) annotations = list(paths[1]) assert type(inp_images) is list assert type(annotations) is list tp = np.zeros(model.n_classes) fp = np.zeros(model.n_classes) fn = np.zeros(model.n_classes) n_pixels = np.zeros(model.n_classes) for inp, ann in tqdm(zip(inp_images, annotations)): pr = predict(model, inp) gt = get_segmentation_array(ann, model.n_classes, model.output_width, model.output_height, no_reshape=True) gt = gt.argmax(-1) pr = pr.flatten() gt = gt.flatten() for cl_i in range(model.n_classes): tp[cl_i] += np.sum((pr == cl_i) * (gt == cl_i)) fp[cl_i] += np.sum((pr == cl_i) * ((gt != cl_i))) fn[cl_i] += np.sum((pr != cl_i) * ((gt == cl_i))) n_pixels[cl_i] += np.sum(gt == cl_i) cl_wise_score = tp / (tp + fp + fn + 0.000000000001) n_pixels_norm = n_pixels / np.sum(n_pixels) frequency_weighted_IU = np.sum(cl_wise_score * n_pixels_norm) mean_IU = np.mean(cl_wise_score) return { "frequency_weighted_IU": frequency_weighted_IU, "mean_IU": mean_IU, "class_wise_IU": cl_wise_score }
def evaluate(model=None, inp_images=None, annotations=None, checkpoints_path=None): ious = [] accs = [] if not checkpoints_path is None: model = model_from_checkpoint_path(checkpoints_path) image_seg_pairs = get_pairs_from_paths(inp_images, annotations) for inp, ann in image_seg_pairs: pr = predict(model, inp).reshape(model.output_height * model.output_width) if np.isnan(pr).any(): print("got nan on %s" % inp) continue gt = get_segmentation_arr(ann, model.n_classes, model.output_width, model.output_height) gt = gt.argmax(-1) iou = get_iou(gt, pr, model.n_classes) acc = get_binary_accuracy(gt, pr) ious.append(iou) accs.append(acc) ious = np.array(ious) print("Class wise IoU ", np.mean(ious, axis=0)) print("Mean IoU ", np.mean(ious)) print("Mean Acc", np.mean(accs))
def evaluate_segmentation(model=None, inp_images=None, annotations=None, inp_images_dir=None, annotations_dir=None, checkpoints_path=None, zero_handling="None", split_factor=1): #, optimizer='adadelta', loss='categorical_crossentropy', metrics=['accuracy']): if model is None: assert (checkpoints_path is not None),\ "Please provide the model or the checkpoints_path" model = model_from_checkpoint_path(checkpoints_path) if inp_images is None: assert (inp_images_dir is not None),\ "Please provide inp_images or inp_images_dir" assert (annotations_dir is not None),\ "Please provide inp_images or inp_images_dir" paths = get_pairs_from_paths(inp_images_dir, annotations_dir, split_factor=split_factor) paths = list(zip(*paths)) inp_images = list(paths[0]) annotations = list(paths[1]) assert type(inp_images) is list assert type(annotations) is list tp = np.zeros(model.n_classes - 1) fp = np.zeros(model.n_classes - 1) fn = np.zeros(model.n_classes - 1) n_pixels = np.zeros(model.n_classes - 1) for inp, ann in tqdm(zip(inp_images, annotations)): pr = predict(model, inp) gt = get_segmentation_array(ann, model.n_classes, model.output_width, model.output_height, no_reshape=True) gt = gt.argmax(-1) pr = pr.flatten() gt = gt.flatten() for cl_i in range(model.n_classes): # Ignoring zero class (ambiguous) if cl_i > 0: i = cl_i - 1 tp[i] += np.sum((pr == cl_i) * (gt == cl_i)) fp[i] += np.sum((pr == cl_i) * ((gt != cl_i))) fn[i] += np.sum((pr != cl_i) * ((gt == cl_i))) n_pixels[i] += np.sum(gt == cl_i) zero_total = np.sum(gt == 0) zero_positive = np.sum((pr == cl_i) * (gt == 0)) #zero_negative = np.sum((pr != cl_i) * (gt == 0)) # unique, counts = np.unique(gt, return_counts=True) # occurs = dict(zip(unique, counts)) # If zero_handling is none of these, it will be worst case, # where zero classes are always classified as wrong # if zero_handling == "MeanOnPrevData": # # Zero class has a weight of 1/(num_classes-1) # tp[i] += 0.5*occurs[0]/(model.n_classes-1) # fp[i] -= occurs[0]/(model.n_classes-1) # fn[i] += 0.5*occurs[0]/(model.n_classes-1) # n_pixels[i] += occurs[0]/(model.n_classes-1) # elif zero_handling == "WeightedOnPrevData": # # Zero class has a weight that is proportional to # # number of occurences of a certain prediction # tp_score = tp[i] / (tp[i] + fn[i] + 0.000000000001) # fn_score = fn[i] / (tp[i] + fn[i] + 0.000000000001) # tp[i] += tp_score*occurs[0]/(model.n_classes-1) # fp[i] -= occurs[0]/(model.n_classes-1) # fn[i] += fn_score*occurs[0]/(model.n_classes-1) # n_pixels[i] += occurs[0]/(model.n_classes-1) # elif zero_handling == "TrustfulOnPrevData": # # Zero class has always weight 1 for every prediction # tp[i] += occurs[0]/(model.n_classes-1) # fp[i] -= occurs[0]/(model.n_classes-1) # #fn[i] += 0.5*occurs[0] # n_pixels[i] += occurs[0]/(model.n_classes-1) if zero_handling == "Mean": # Zero class has a weight of 1/(num_classes-1) tp[i] += 0.5 * zero_positive fp[i] -= zero_positive fn[i] += 0.5 * zero_positive n_pixels[i] += zero_positive elif zero_handling == "Weighted": tp_score = zero_positive / (zero_total + 0.000000000001) tp[i] += zero_positive * tp_score fp[i] -= zero_positive fn[i] += zero_positive * (1 - tp_score) n_pixels[i] += zero_positive elif zero_handling == "Trustful": tp[i] += zero_positive fp[i] -= zero_positive #fn[i] += zero_negative n_pixels[i] += zero_positive #elif zero_handling == "None": # elif zero_handling == "Idealistic": # conf_score = (zero_positive / (zero_total + 0.000000000001)) # tp[i] += occurs[0]*conf_score # fp[i] -= occurs[0]*conf_score # #fn[i] += zero_positive*(conf_score-1) # n_pixels[i] += occurs[0]*conf_score cl_wise_score = tp / (tp + fp + fn + 0.000000000001) n_pixels_norm = n_pixels / np.sum(n_pixels) frequency_weighted_IU = np.sum(cl_wise_score * n_pixels_norm) mean_IU = np.mean(cl_wise_score) #model.compile(optimizer=optimizer, loss=loss, metrics=metrics) #results = model.evaluate(inp_images_dir, annotations_dir) return { #"loss": results[0], #"accuracy": results[1], "frequency_weighted_IoU": frequency_weighted_IU, "mean_IoU": mean_IU, "class_wise_IoU": cl_wise_score, "pixels_per_class": n_pixels }