def wbc(bboxes, scores, thr=0.2):
# bboxes and score must be tensors
if isinstance(bboxes, list):
device = bboxes[0].device
bboxes = torch.stack(bboxes).to("cpu")
else:
device = bboxes.device
bboxes = bboxes.to("cpu")
if isinstance(scores, list):
scores = torch.stack(scores).to("cpu")
else:
scores = scores.to("cpu")
# change the order from xywh to xyxy
bboxes = np.asarray(change_box_order(bboxes, order="xywh2xyxy"))
scores = np.asarray(scores)
# perform (my version) of weighted box clustering
wbc_bboxes, wbc_scores = weighted_box_clustering(bboxes, scores, thr=thr)
# # filter out overlapping boxes
# wbc_bboxes, wbc_scores = rm_overlapping_boxes(torch.tensor(wbc_bboxes, dtype=torch.float32),
# torch.tensor(wbc_scores, dtype=torch.float32))
# convert the boxes in the "xywh" form again
wbc_bboxes = change_box_order(torch.tensor(wbc_bboxes, dtype=torch.float32),
order="xyxy2xywh").to(device)
wbc_scores = torch.tensor(wbc_scores, dtype=torch.float32).to(device)
return wbc_bboxes, wbc_scores
def calc_detection_hits(gt_bbox, crop_bbox, score_bbox, iou_thr = 0.2, score_thr=0.):
# bboxes and score must be tensors
if isinstance(crop_bbox, list):
crop_bbox = torch.stack(crop_bbox)
if isinstance(score_bbox, list):
score_bbox = torch.stack(score_bbox)
# sort according to score and limit according to score_thr
scores, indices = torch.sort(score_bbox, descending=True)
boxes = crop_bbox[indices]
boxes = boxes[scores > score_thr]
scores = scores[scores > score_thr]
# calculate iou
iou_matrix = box_iou(change_box_order(gt_bbox, order='xywh2xyxy'),
change_box_order(boxes, order="xywh2xyxy"))
# create label list, denoting at max one detection per lesion
labels = torch.zeros_like(scores)
iou_hits = torch.zeros_like(scores)
for i in range(iou_matrix.shape[0]):
hits = (iou_matrix[i] > iou_thr).nonzero()
if len(hits) > 0:
labels[hits[0].cpu()] = 1.
iou_hits[hits[0].cpu()] = iou_matrix[i, hits[0].cpu()]
return labels, scores, iou_hits
def forward(self, data, loc_preds, loc_targets, cls_preds, cls_targets):
batch_size = loc_preds.shape[0]
anchors = Anchors()
labels = []
preds = []
for i in range(batch_size):
pred_boxes, pred_labels, pred_score = anchors.generateBoxesFromAnchors(
loc_preds[i],
cls_preds[i],
tuple(data[i].shape[1:]),
cls_tresh=0.05)
#start = time.time()
target_boxes, target_labels = anchors.restoreBoxesFromAnchors(
loc_targets[i], cls_targets[i], tuple(data[i].shape[1:]))
#end = time.time()
#print(end-start)
if pred_boxes is None and target_boxes is None:
continue
if pred_boxes is None:
preds.append(torch.zeros_like(target_labels))
labels.append(target_labels)
continue
if target_boxes is None:
preds.append(pred_labels)
labels.append(torch.zeros_like(pred_labels))
continue
pred_boxes = change_box_order(pred_boxes, order='xywh2xyxy')
target_boxes = change_box_order(target_boxes, order='xywh2xyxy')
iou_matrix = box_iou(target_boxes, pred_boxes)
iou_matrix = iou_matrix > self.iou_thr
box_labels = torch.clamp(torch.sum(iou_matrix, 0), 0, 1)
preds.append(pred_score)
labels.append(box_labels)
labels = torch.tensor([item for sublist in labels for item in sublist])\
.type(torch.float32)
preds = torch.tensor([item for sublist in preds for item in sublist])\
.type(torch.float32)
if not any(labels):
return float_to_tensor(0.5)
elif all(labels):
return float_to_tensor(1)
elif labels.dim() == 0:
return float_to_tensor(0)
else:
return float_to_tensor(roc_auc_score(labels, preds))
def gt_overlap(image_bbox, crop_bbox, iou_thr = 0.2):
# convert bbox format from [x, y, w, h] to [x_1, y_1, x_2, y_2] for IoU
# calculation
image_bbox = change_box_order(image_bbox, order='xywh2xyxy')
crop_bbox = change_box_order(crop_bbox, order="xywh2xyxy")
# determine overlap with ground truth bboxes
iou_matrix = box_iou(image_bbox, crop_bbox)
iou_matrix = iou_matrix > iou_thr
# assign each detected bbox a label: 1 if its overlap with the ground
# truth is higher than the given threshold, 0 otherwise
box_class = torch.clamp(torch.sum(iou_matrix, 0), 0, 1)
return box_class
def nms(bboxes, scores, labels=None, thr=0.2):
# merge overlapping bounding boxes
# bboxes and score must be tensors
if isinstance(bboxes, list):
bboxes = torch.stack(bboxes)
if isinstance(scores, list):
scores = torch.stack(scores)
# sort the score in descending order, adjust bboxes accordingly
scores, indices = torch.sort(scores, descending=True)
bboxes = bboxes[indices]
if labels is not None:
if isinstance(labels, list):
labels = torch.tensor(labels)
labels = labels[indices]
# change the order from xywh to xyxy
bboxes = change_box_order(bboxes, order="xywh2xyxy")
# limit the amount of bboxes to 300 (or less)
if len(scores) > 300:
limit = 300
else:
limit = len(scores)
scores = scores[0:limit]
bboxes = bboxes[0:limit]
if labels is not None:
labels = labels[0:limit]
# perform an image-wise NMS
keep_ids = box_nms(bboxes, scores, threshold=thr)
bboxes = bboxes[keep_ids]
scores = scores[keep_ids]
if labels is not None:
labels = labels[keep_ids]
else:
labels = []
# filter out overlapping bboxes
#bboxes, scores, labels = rm_overlapping_boxes(bboxes, scores, labels=labels)
# convert in xywh form again
bboxes = change_box_order(bboxes, order="xyxy2xywh")
return bboxes, scores, labels
def calc_tp_fn_fp(gt_bbox, crop_bbox, score_bbox, iou_thr = 0.2,
confidence_values = np.arange(0.5, 1, 0.05)):
# bboxes and score must be tensors
if isinstance(crop_bbox, list):
crop_bbox = torch.stack(crop_bbox)
if isinstance(score_bbox, list):
score_bbox = torch.stack(score_bbox)
tp_list = []
fp_list = []
fn_list = []
iou_list = []
gt_bbox = change_box_order(gt_bbox, order='xywh2xyxy')
for j in confidence_values:
current_bbox = crop_bbox[score_bbox > j]
if len(current_bbox) == 0:
tp_list.append(torch.tensor(0, device = current_bbox.device))
fp_list.append(torch.tensor(0, device = current_bbox.device))
fn_list.append(torch.tensor(gt_bbox.shape[0],
device = current_bbox.device))
continue
# break
iou_matrix = box_iou(gt_bbox,
change_box_order(current_bbox,
order="xywh2xyxy"))
hits = iou_matrix > iou_thr
iou_values = iou_matrix[hits]
iou_list.append(iou_values)
# true positives are the lesions that are recognized
# count only one detected box per lesion as positive
tp = torch.clamp(torch.sum(hits, 1), 0, 1).sum()
tp_list.append(tp)
# false negatives are the lesions that are NOT recognized
fn = gt_bbox.shape[0] - tp
fn_list.append(fn)
# number of false positives
fp = (current_bbox.shape[0] - tp).type(torch.float32)
fp_list.append(fp)
return tp_list, fp_list, fn_list
def rm_overlapping_boxes(bboxes, scores, labels=None, order="xyxy"):
if order == "xywh":
bboxes = change_box_order(bboxes, order="xywh2xyxy")
scores, indices = torch.sort(scores, descending=True)
bboxes = bboxes[indices]
if labels is not None:
labels = labels[indices]
result_labels = [labels[0]]
# filter out overlapping bboxes
result_bboxes = [bboxes[0]]
result_scores = [scores[0]]
for i in range(1, len(bboxes)):
ignore = False
for j in range(len(result_bboxes)):
# if i == j:
# continue
if box_overlap(bboxes[i], result_bboxes[j]):
ignore = True
break
if not ignore:
result_bboxes.append(bboxes[i])
result_scores.append(scores[i])
if labels is not None:
result_labels.append(labels[i])
result_bboxes = torch.stack(result_bboxes)
result_scores = torch.stack(result_scores)
if order == "xywh":
result_bboxes = change_box_order(result_bboxes, order="xyxy2xywh")
if labels is not None:
return result_bboxes, result_scores, torch.stack(result_labels)
else:
return result_bboxes, result_scores, None
def eval(dataset, model_path, plot=False):
# device
device = 'cuda'
# load model
checkpoint = torch.load(model_path)
model = RetinaNet(**checkpoint['init_kwargs']).eval()
model.load_state_dict(checkpoint['state_dict'])
model.to(device)
# hyperparams
crop_size = [600, 600]
overlapped_boxes = 0.5
confidence_values = np.arange(0.5, 1, 0.05)
tpr_list = []
fppi_list = []
with torch.no_grad():
for i in tqdm(range(len(dataset))):
torch.cuda.empty_cache()
# get image data
test_data = dataset[i]
# crop background
test_data = inbreast_utils.segment_breast(test_data)
image_bbox = utils.bounding_box(dataset[i]["seg"])
# generate crops
crop_list, corner_list = inbreast_utils.create_crops(test_data)
# define list for predicted bboxes in crops
crop_bbox = []
score_bbox = []
# plot the image with the according bboxes
if plot:
# plot image
plt.figure(1, figsize=(15, 10))
fig, ax = plt.subplots(1)
ax.imshow(test_data["data"][0, :, :], cmap='Greys_r')
# show bboxes as saved in data (in red with center)
for l in range(len(image_bbox)):
pos = tuple(image_bbox[l][0:2])
plt.plot(pos[0], pos[1], 'r.')
width = image_bbox[l][2]
height = image_bbox[l][3]
pos = (pos[0] - np.floor(width / 2),
pos[1] - np.floor(height / 2))
# Create a Rectangle patch
rect = patches.Rectangle(pos,
width,
height,
linewidth=1,
edgecolor='r',
facecolor='none')
ax.add_patch(rect)
# iterate over crops
for j in tqdm(range(0, len(crop_list))):
torch.cuda.empty_cache()
test_image = torch.Tensor(crop_list[j]['data']).to(device)
test_bbox = utils.bounding_box(crop_list[j]['seg'])
# predict anchors and labels for the crops using the loaded model
anchor_preds, cls_preds = model(test_image.unsqueeze(0))
# convert the predicted anchors to bboxes
anchors = Anchors()
boxes, labels, score = anchors.generateBoxesFromAnchors(
anchor_preds[0].to('cpu'),
cls_preds[0].to('cpu'),
tuple(test_image.shape[1:]),
cls_tresh=0.05)
# correct the predicted bboxes
for k in range(len(boxes)):
center_corrected = boxes[k][0:2] + \
torch.Tensor(corner_list[j])
crop_bbox.append(
torch.cat((center_corrected, boxes[k][2:])))
score_bbox.append(score[k])
# merge overlapping bounding boxes
crop_bbox, score_bbox = merge(crop_bbox, score_bbox)
# calculate the FROC metric (TPR vs. FPPI)
tpr_int = []
fppi_int = []
image_bbox = change_box_order(torch.Tensor(image_bbox),
order='xywh2xyxy').to('cpu')
iou_thr = 0.2
for j in confidence_values:
current_bbox = crop_bbox[score_bbox > j]
if len(current_bbox) == 0:
tpr_int.append(torch.Tensor([0]))
fppi_int.append(torch.Tensor([0]))
continue
#break
iou_matrix = box_iou(
image_bbox,
change_box_order(current_bbox, order="xywh2xyxy"))
iou_matrix = iou_matrix > iou_thr
# true positives are the lesions that are recognized
tp = iou_matrix.sum()
# false negatives are the lesions that are NOT recognized
fn = image_bbox.shape[0] - tp
# true positive rate
tpr = tp.type(torch.float32) / (tp + fn).type(torch.float32)
tpr = torch.clamp(tpr, 0, 1)
# number of false positives per image
fp = (current_bbox.shape[0] - tp).type(torch.float32)
tpr_int.append(tpr)
fppi_int.append(fp)
tpr_list.append(tpr_int)
fppi_list.append(fppi_int)
if plot:
# show the predicted bboxes (in blue)
print("Number of detected bboxes: {0}".format(len(crop_bbox)))
keep = score_bbox > 0.5
crop_bbox = crop_bbox[keep]
score_bbox = score_bbox[keep]
for j in range(len(crop_bbox)):
width = crop_bbox[j][2]
height = crop_bbox[j][3]
pos = (crop_bbox[j][0] - torch.floor(width / 2),
crop_bbox[j][1] - torch.floor(height / 2))
# Create a Rectangle patch
rect = patches.Rectangle(pos,
width,
height,
linewidth=1,
edgecolor='b',
facecolor='none')
ax.add_patch(rect)
ax.annotate("{:.2f}".format(score_bbox[j]),
pos,
fontsize=6,
xytext=(pos[0] + 10, pos[1] - 10))
print("BBox params: {0}, score: {1}".format(
crop_bbox[j], score_bbox[j]))
plt.show()
# fig.savefig("../plots/" + "_".join(model_path.split("/")[5:8]) + ".png")
# calculate FROC over all test images
tpr_list = np.asarray(tpr_list)
tpr = np.sum(tpr_list, axis=0) / tpr_list.shape[0]
fppi_list = np.asarray(fppi_list)
fppi = np.sum(fppi_list, axis=0) / fppi_list.shape[0]
# plt.figure(1)
# plt.ylim(0, 1.1)
# plt.xlabel("False Positve per Image (FPPI)")
# plt.ylabel("True Positive Rate (TPR)")
# plt.title("Free Response Operating Characteristic (FROC)")
# plt.plot(np.asarray(fppi), np.asarray(tpr), "rx-")
# plt.show()
return tpr, fppi
def merge_jung(bboxes, scores, merge_thr=0.2):
def iteration(annotation_list):
total_set_list = []
merged_list = []
for annotation in annotation_list:
if len(total_set_list) == 0:
total_set_list.append([annotation])
else:
isAppended = False
for set in total_set_list:
for saved_annotation in set:
iou = box_iou(saved_annotation[0].view(1, -1),
annotation[0].view(1, -1),
"xyxy")
isCovered = isInside(saved_annotation[0],
annotation[0])
#if iou > merge_thr or isCovered:
if iou > 0.2:
set.append(annotation)
isAppended = True
break
if isAppended == True:
break
if isAppended == False:
total_set_list.append([annotation])
for set in total_set_list:
value = 0
max_area = 0
max_info = None
max_value = 0
for idx, saved_annotation in enumerate(set):
area = (saved_annotation[0][3] - saved_annotation[0][1]) * \
(saved_annotation[0][2] - saved_annotation[0][0])
value += saved_annotation[1]
if area > max_area:
max_area = area
max_info = saved_annotation
if float(saved_annotation[1]) > max_value:
max_value = float(saved_annotation[1])
value = float(value) / len(set)
merged_list.append((max_info[0], max_value))
return merged_list
# bboxes and score must be tensors
if isinstance(bboxes, list):
bboxes = torch.stack(bboxes)
if isinstance(scores, list):
#scores = np.asarray(torch.stack(scores))
scores = torch.stack(scores)
# change the order from xywh to xyxy
bboxes = change_box_order(bboxes, order="xywh2xyxy")
# save boxes and scores in required format
raw_list = [[bboxes[i], scores[i]] for i in range(len(bboxes))]
# remove overlapping boxes (first run)
final_list = iteration(raw_list)
# remove remaining overlapping boxes till no box can be merged anymore
change = True
while change:
length = len(final_list)
final_list = iteration(final_list)
if length == len(final_list):
change = False
bboxes = [final_list[i][0] for i in range(len(final_list))]
bboxes = torch.stack(bboxes)
scores = [final_list[i][1] for i in range(len(final_list))]
scores = torch.tensor(scores, dtype=torch.float32)
# filter out overlapping bboxes
#bboxes, scores = rm_overlapping_boxes(bboxes, scores)
# change box order
bboxes = change_box_order(bboxes, order="xyxy2xywh")
# sort the score in descending order, adjust bboxes accordingly
scores, indices = torch.sort(scores, descending=True)
bboxes = bboxes[indices]
return bboxes, scores
def my_merging_2(bboxes, scores, crop_center_factor, heatmap_factor, thr=0.2):
# bboxes and score must be tensors
if isinstance(bboxes, list):
device = bboxes[0].device
bboxes = torch.stack(bboxes).to("cpu")
else:
device = bboxes.device
bboxes = bboxes.to("cpu")
if isinstance(scores, list):
scores = torch.stack(scores).to("cpu")
# sort the score in descending order, adjust bboxes accordingly
scores, indices = torch.sort(scores, descending=True)
bboxes = bboxes[indices]
# change the order from xywh to xyxy
bboxes = change_box_order(bboxes, order="xywh2xyxy")
box_set_list = []
for i in range(len(bboxes)):
if len(box_set_list) == 0:
box_set_list.append([[bboxes[i], scores[i]]])
else:
isAppended = False
for box_set in box_set_list:
for saved_box in box_set:
iou = box_iou(saved_box[0].view(1, -1),
bboxes[i].view(1, -1),
"xyxy")
isCovered = isInside(saved_box[0],
bboxes[i])
if iou > thr or isCovered:
# if iou > 0.2:
box_set.append([bboxes[i], scores[i]])
isAppended = True
break
if isAppended == True:
break
if isAppended == False:
box_set_list.append([[bboxes[i], scores[i]]])
merged_box_list = []
merged_score_list = []
for i in range(len(box_set_list)):
pos = [box_set_list[i][j][0] for j in range(len(box_set_list[i]))]
scores = [box_set_list[i][j][1] for j in range(len(box_set_list[i]))]
pos = torch.stack(pos)
scores = torch.stack(scores)
xx1 = torch.max(pos[0, 0], pos[:,0])
yy1 = torch.max(pos[0, 1], pos[:,1])
xx2 = torch.min(pos[0, 2], pos[:,2])
yy2 = torch.min(pos[0, 3], pos[:,3])
w = torch.max(torch.Tensor([0]), xx2 - xx1 + 1)
h = torch.max(torch.Tensor([0]), yy2 - yy1 + 1)
inter = w * h
areas = (pos[:,2] - pos[:,0] + 1) * (pos[:,3] - pos[:,1] + 1)
# overall between currently highest scoring box and all boxes.
iou = inter / (areas[0] + areas - inter)
match_weigths = iou
match_scores = scores * match_weigths
avg_score = torch.sum(match_scores) / torch.sum(match_weigths)
merged_score_list.append(avg_score)
avg_pos = torch.tensor([torch.sum(pos[:,0] * match_scores) / torch.sum(match_scores),
torch.sum(pos[:, 1] * match_scores) / torch.sum(match_scores),
torch.sum(pos[:, 2] * match_scores) / torch.sum(match_scores),
torch.sum(pos[:, 3] * match_scores) / torch.sum(match_scores)])
merged_box_list.append(avg_pos)
# convert the boxes in the "xywh" form and tensors again
keep_bboxes = change_box_order(torch.stack(merged_box_list),
order="xyxy2xywh").to(device)
keep_scores = torch.tensor(merged_score_list, dtype=torch.float32).to(device)
return keep_bboxes, keep_scores
def my_merging(bboxes, scores, crop_center_factor, heatmap_factor, thr=0.2):
# bboxes and score must be tensors
if isinstance(bboxes, list):
device = bboxes[0].device
bboxes = torch.stack(bboxes).to("cpu")
else:
device = bboxes.device
bboxes = bboxes.to("cpu")
if isinstance(scores, list):
scores = np.asarray(torch.stack(scores).to("cpu"))
if isinstance(crop_center_factor, list):
crop_center_factor = np.asarray(crop_center_factor)
if isinstance(heatmap_factor, list):
heatmap_factor = np.asarray(heatmap_factor)
# change the order from xywh to xyxy
bboxes = np.asarray(change_box_order(bboxes, order="xywh2xyxy"))
# order is the sorted index. maps order to index o[1] = 24 (rank1, ix 24)
order = scores.argsort()[::-1]
# limit the amount of bboxes to 300 (or less)
if len(order) > 300:
limit = 300
else:
limit = len(order)
order = order[:limit]
bboxes = bboxes[order]
scores = scores[order]
crop_center_factor = crop_center_factor[order]
heatmap_factor = heatmap_factor[order]
order = scores.argsort()[::-1]
# define list for bboxes to keep
keep_scores = []
keep_bboxes = []
# seperate coordinates
x1 = bboxes[:, 0]
y1 = bboxes[:, 1]
x2 = bboxes[:, 2]
y2 = bboxes[:, 3]
# calculate the area of each box
areas = (y2 - y1 + 1) * (x2 - x1 + 1)
while order.size > 0:
i = order[0] # higehst scoring element
xx1 = np.maximum(x1[i], x1[order])
yy1 = np.maximum(y1[i], y1[order])
xx2 = np.minimum(x2[i], x2[order])
yy2 = np.minimum(y2[i], y2[order])
w = np.maximum(0.0, xx2 - xx1 + 1)
h = np.maximum(0.0, yy2 - yy1 + 1)
inter = w * h
# overall between currently highest scoring box and all boxes.
iou = inter / (areas[i] + areas[order] - inter)
# get all the predictions that match the current box to build one cluster.
#criterium = (iou > thr) | (inter / areas[order] > 0.5)
criterium = (iou > thr)
matches = np.argwhere(criterium)
match_iou = iou[matches]
match_areas = areas[order[matches]]
match_scores = scores[order[matches]]
match_ccf = crop_center_factor[order[matches]]
match_hf = heatmap_factor[order[matches]]
#print(match_scores)
# weight all scores in cluster by patch factors, and size.
match_score_weights = match_iou * match_ccf # / match_hf # * match_areas
#match_score_weights = np.ones_like(match_scores)
#match_score_weights = match_iou
match_scores *= match_score_weights
# compute weighted average score for the cluster
avg_score = np.sum(match_scores) / np.sum(match_score_weights)
#avg_score = np.sum(match_scores) / len(match_scores)
# compute weighted average of coordinates for the cluster. now only take existing
# predictions into account.
avg_coords = [
np.sum(x1[order[matches]] * match_scores) / np.sum(match_scores),
np.sum(y1[order[matches]] * match_scores) / np.sum(match_scores),
np.sum(x2[order[matches]] * match_scores) / np.sum(match_scores),
np.sum(y2[order[matches]] * match_scores) / np.sum(match_scores)]
# some clusters might have very low scores due to high amounts of missing predictions.
# filter out the with a conservative threshold, to speed up evaluation.
if avg_score > 0.05:
keep_scores.append(avg_score)
keep_bboxes.append(avg_coords)
# get index of all elements that were not matched and discard all others.
inds = np.where(~criterium)[0]
order = order[inds]
# convert the boxes in the "xywh" form and tensors again
keep_bboxes = change_box_order(torch.tensor(keep_bboxes,
dtype=torch.float32),
order="xyxy2xywh").to(device)
keep_scores = torch.tensor(keep_scores, dtype=torch.float32).to(device)
return keep_bboxes, keep_scores
