def evaluate(model=None,
inp_images=None,
annotations=None,
inp_images_dir=None,
annotations_dir=None,
checkpoints_path=None):
if model is None:
assert (
checkpoints_path
is not None), "Please provide the model or the checkpoints_path"
model = model_from_checkpoint_path(checkpoints_path)
if inp_images is None:
assert (inp_images_dir
is not None), "Please privide inp_images or inp_images_dir"
assert (annotations_dir
is not None), "Please privide inp_images or inp_images_dir"
paths = get_pairs_from_paths(inp_images_dir, annotations_dir)
paths = list(zip(*paths))
inp_images = list(paths[0])
annotations = list(paths[1])
assert type(inp_images) is list
assert type(annotations) is list
tp = np.zeros(model.n_classes)
fp = np.zeros(model.n_classes)
fn = np.zeros(model.n_classes)
n_pixels = np.zeros(model.n_classes)
for inp, ann in tqdm(zip(inp_images, annotations)):
pr = predict(model, inp)
gt = get_segmentation_array(ann,
model.n_classes,
model.output_width,
model.output_height,
no_reshape=True)
gt = gt.argmax(-1)
pr = pr.flatten()
gt = gt.flatten()
for cl_i in range(model.n_classes):
tp[cl_i] += np.sum((pr == cl_i) * (gt == cl_i))
fp[cl_i] += np.sum((pr == cl_i) * ((gt != cl_i)))
fn[cl_i] += np.sum((pr != cl_i) * ((gt == cl_i)))
n_pixels[cl_i] += np.sum(gt == cl_i)
cl_wise_score = tp / (tp + fp + fn + 0.000000000001)
n_pixels_norm = n_pixels / np.sum(n_pixels)
frequency_weighted_IU = np.sum(cl_wise_score * n_pixels_norm)
mean_IU = np.mean(cl_wise_score)
return {
"frequency_weighted_IU": frequency_weighted_IU,
"mean_IU": mean_IU,
"class_wise_IU": cl_wise_score
}
def evaluate(model=None, inp_images=None, annotations=None, checkpoints_path=None):
ious = []
accs = []
if not checkpoints_path is None:
model = model_from_checkpoint_path(checkpoints_path)
image_seg_pairs = get_pairs_from_paths(inp_images, annotations)
for inp, ann in image_seg_pairs:
pr = predict(model, inp).reshape(model.output_height * model.output_width)
if np.isnan(pr).any():
print("got nan on %s" % inp)
continue
gt = get_segmentation_arr(ann, model.n_classes, model.output_width, model.output_height)
gt = gt.argmax(-1)
iou = get_iou(gt, pr, model.n_classes)
acc = get_binary_accuracy(gt, pr)
ious.append(iou)
accs.append(acc)
ious = np.array(ious)
print("Class wise IoU ", np.mean(ious, axis=0))
print("Mean IoU ", np.mean(ious))
print("Mean Acc", np.mean(accs))
def evaluate_segmentation(model=None,
inp_images=None,
annotations=None,
inp_images_dir=None,
annotations_dir=None,
checkpoints_path=None,
zero_handling="None",
split_factor=1):
#, optimizer='adadelta', loss='categorical_crossentropy', metrics=['accuracy']):
if model is None:
assert (checkpoints_path is not None),\
"Please provide the model or the checkpoints_path"
model = model_from_checkpoint_path(checkpoints_path)
if inp_images is None:
assert (inp_images_dir is not None),\
"Please provide inp_images or inp_images_dir"
assert (annotations_dir is not None),\
"Please provide inp_images or inp_images_dir"
paths = get_pairs_from_paths(inp_images_dir,
annotations_dir,
split_factor=split_factor)
paths = list(zip(*paths))
inp_images = list(paths[0])
annotations = list(paths[1])
assert type(inp_images) is list
assert type(annotations) is list
tp = np.zeros(model.n_classes - 1)
fp = np.zeros(model.n_classes - 1)
fn = np.zeros(model.n_classes - 1)
n_pixels = np.zeros(model.n_classes - 1)
for inp, ann in tqdm(zip(inp_images, annotations)):
pr = predict(model, inp)
gt = get_segmentation_array(ann,
model.n_classes,
model.output_width,
model.output_height,
no_reshape=True)
gt = gt.argmax(-1)
pr = pr.flatten()
gt = gt.flatten()
for cl_i in range(model.n_classes):
# Ignoring zero class (ambiguous)
if cl_i > 0:
i = cl_i - 1
tp[i] += np.sum((pr == cl_i) * (gt == cl_i))
fp[i] += np.sum((pr == cl_i) * ((gt != cl_i)))
fn[i] += np.sum((pr != cl_i) * ((gt == cl_i)))
n_pixels[i] += np.sum(gt == cl_i)
zero_total = np.sum(gt == 0)
zero_positive = np.sum((pr == cl_i) * (gt == 0))
#zero_negative = np.sum((pr != cl_i) * (gt == 0))
# unique, counts = np.unique(gt, return_counts=True)
# occurs = dict(zip(unique, counts))
# If zero_handling is none of these, it will be worst case,
# where zero classes are always classified as wrong
# if zero_handling == "MeanOnPrevData":
# # Zero class has a weight of 1/(num_classes-1)
# tp[i] += 0.5*occurs[0]/(model.n_classes-1)
# fp[i] -= occurs[0]/(model.n_classes-1)
# fn[i] += 0.5*occurs[0]/(model.n_classes-1)
# n_pixels[i] += occurs[0]/(model.n_classes-1)
# elif zero_handling == "WeightedOnPrevData":
# # Zero class has a weight that is proportional to
# # number of occurences of a certain prediction
# tp_score = tp[i] / (tp[i] + fn[i] + 0.000000000001)
# fn_score = fn[i] / (tp[i] + fn[i] + 0.000000000001)
# tp[i] += tp_score*occurs[0]/(model.n_classes-1)
# fp[i] -= occurs[0]/(model.n_classes-1)
# fn[i] += fn_score*occurs[0]/(model.n_classes-1)
# n_pixels[i] += occurs[0]/(model.n_classes-1)
# elif zero_handling == "TrustfulOnPrevData":
# # Zero class has always weight 1 for every prediction
# tp[i] += occurs[0]/(model.n_classes-1)
# fp[i] -= occurs[0]/(model.n_classes-1)
# #fn[i] += 0.5*occurs[0]
# n_pixels[i] += occurs[0]/(model.n_classes-1)
if zero_handling == "Mean":
# Zero class has a weight of 1/(num_classes-1)
tp[i] += 0.5 * zero_positive
fp[i] -= zero_positive
fn[i] += 0.5 * zero_positive
n_pixels[i] += zero_positive
elif zero_handling == "Weighted":
tp_score = zero_positive / (zero_total + 0.000000000001)
tp[i] += zero_positive * tp_score
fp[i] -= zero_positive
fn[i] += zero_positive * (1 - tp_score)
n_pixels[i] += zero_positive
elif zero_handling == "Trustful":
tp[i] += zero_positive
fp[i] -= zero_positive
#fn[i] += zero_negative
n_pixels[i] += zero_positive
#elif zero_handling == "None":
# elif zero_handling == "Idealistic":
# conf_score = (zero_positive / (zero_total + 0.000000000001))
# tp[i] += occurs[0]*conf_score
# fp[i] -= occurs[0]*conf_score
# #fn[i] += zero_positive*(conf_score-1)
# n_pixels[i] += occurs[0]*conf_score
cl_wise_score = tp / (tp + fp + fn + 0.000000000001)
n_pixels_norm = n_pixels / np.sum(n_pixels)
frequency_weighted_IU = np.sum(cl_wise_score * n_pixels_norm)
mean_IU = np.mean(cl_wise_score)
#model.compile(optimizer=optimizer, loss=loss, metrics=metrics)
#results = model.evaluate(inp_images_dir, annotations_dir)
return {
#"loss": results[0],
#"accuracy": results[1],
"frequency_weighted_IoU": frequency_weighted_IU,
"mean_IoU": mean_IU,
"class_wise_IoU": cl_wise_score,
"pixels_per_class": n_pixels
}