def to_rotated_boxes(boxes, oriens):
boxes_with_oriens = torch.cat((boxes, oriens), dim=-1)
# print(boxes)
for j, boxnorien in enumerate(boxes_with_oriens):
# boxnorien = boxnorien.squeeze_()
alpha = torch.atan2(boxnorien[:][-2],
boxnorien[:][-1]) * 180 / 3.1415926
# alpha = alpha.squeeze_()
# print(alpha,anntype,'====')
# top_left, bottom_right = box[:2].tolist(), box[2:].tolist()
top_left, bottom_right = boxnorien[:2], boxnorien[2:4]
l = bottom_right[1] - top_left[1]
w = bottom_right[0] - top_left[0]
xc = (top_left[0] + bottom_right[0]) / 2
yc = (top_left[1] + bottom_right[1]) / 2
# print(alpha,top_left,bottom_right,'=====')
if l * w <= 1:
continue
box = bBox_2D(l, w, xc, yc, alpha)
box.bBoxCalcVertxex()
print(boxes.size(), oriens.size(), boxnorien.size())
# boxes[j]=torch.
return
def overlay_boxes(image, predictions, anntype):
"""
Adds the predicted boxes on top of the image
Arguments:
image (np.ndarray): an image as returned by OpenCV
predictions (BoxList): the result of the computation by the model.
It should contain the field `labels`.
"""
# labels = predictions.get_field("labels")
oriens = predictions.get_field("rotations")
boxes = predictions.bbox
xclist = []
yclist = []
alphalist = []
# print('\noriens:',oriens.size(),'boxes:',boxes.size(),'==========\n')
for box, orien in zip(boxes, oriens):
color = {'targets': (155, 255, 255), 'output': (155, 255, 55)}
offset = {'targets': 2, 'output': 0}
box = box.squeeze_().detach().cpu().numpy()
alpha = torch.atan2(orien[:][0], orien[:][1]) * 180 / math.pi
alpha = alpha.squeeze_().detach().cpu().numpy()
# print(alpha,anntype,'====')
# top_left, bottom_right = box[:2].tolist(), box[2:].tolist()
top_left, bottom_right = box[:2], box[2:]
l = bottom_right[1] - top_left[1]
w = bottom_right[0] - top_left[0]
xc = (top_left[0] + bottom_right[0]) / 2
yc = (top_left[1] + bottom_right[1]) / 2
xclist.append(xc)
yclist.append(yc)
alphalist.append(alpha)
# if l * w <= 1:
# continue
box = bBox_2D(l, w, xc + offset[anntype], yc + offset[anntype], alpha)
box.bBoxCalcVertxex()
rad = box.alpha * math.pi / 180
cv2.line(image, box.vertex1, box.vertex2, color[anntype], 1, cv2.LINE_AA)
cv2.line(image, box.vertex2, box.vertex4, color[anntype], 1, cv2.LINE_AA)
cv2.line(image, box.vertex3, box.vertex1, color[anntype], 1, cv2.LINE_AA)
cv2.line(image, box.vertex4, box.vertex3, color[anntype], 1, cv2.LINE_AA)
# print(box.vertex4, box.vertex3, box.vertex2, box.vertex1, '====',l*w,'\t',l,'\t',w)
point = int(box.xc - box.length * 0.8 * np.sin(rad)), int(box.yc + box.length * 0.8 * np.cos(rad))
cv2.line(image, (int(box.xc), int(box.yc)),
point,
color[anntype], 1, cv2.LINE_AA)
return np.array([xclist, yclist], dtype=float), np.array(alphalist, dtype=float)
def overlay_GT_on_scan(img, anno, cloudgt, anngt, resolution=1000):
# one img and its ann (ann in pixels)
# GT database of cloudgt and anngt (cloud and ann in original METERs !!!)
# thus, box.resize is needed
point_collide = 0
box_collide = 0
if len(anno) == 0:
return img, anno
ann_num = len(anno)
sampling_num = int(max(
0, (15 - ann_num))) # 12 is the max box num in this dataset
# if ann_num <= 5:
# sampling_num = int(random.random() * max(0, (15 - ann_num))) # 12 is the max box num in this dataset
# else:
# return img, anno
collision_box = False # indicator for collision test
collision_point = False
_pixel_enhance = np.array([-1, 0, 1])
pixel_enhance = np.array([[x, y] for x in _pixel_enhance
for y in _pixel_enhance
]) # enhance pixel by extra 8
existing_box_vertex = []
for ann in anno: # for every existing GT box
label = ann["bbox"]
orien = ann["rotation"]
ebox = bBox_2D(label[3], label[2], label[0] + label[2] / 2,
label[1] + label[3] / 2, orien)
# bBox_2D: length, width, xc, yc,alpha label: 'bbox': [box.xtl, box.ytl, box.width, box.length],
# ebox.resize(1.2)
ebox.bBoxCalcVertxex()
existing_box_vertex.append(
[ebox.vertex1, ebox.vertex2, ebox.vertex3,
ebox.vertex4]) # shape [N,4,2]
existing_box_vertex = np.array(existing_box_vertex).reshape(-1, 2)
img = transforms.ToTensor()(img)
img = img.permute(1, 2, 0)
for num in range(sampling_num): # for every sampled GT box
index = int(random.random() * len(anngt))
ann_sampled = anngt[index]
point_sampled = cloudgt[index]
box = bBox_2D(ann_sampled['bbox'][3], ann_sampled['bbox'][2],
ann_sampled['bbox'][0], ann_sampled['bbox'][1],
ann_sampled['rotation'])
box.scale(900 / 30, 500, 100)
# box.scale(resolution / 200, 0, 0) # converse to img pixel values
# box.resize(1.2) # to ensure points inside
box.bBoxCalcVertxex()
box.xcyc2topleft()
maxx = max(box.vertex1[0], box.vertex2[0], box.vertex3[0],
box.vertex4[0])
minx = min(box.vertex1[0], box.vertex2[0], box.vertex3[0],
box.vertex4[0])
maxy = max(box.vertex1[1], box.vertex2[1], box.vertex3[1],
box.vertex4[1])
miny = min(box.vertex1[1], box.vertex2[1], box.vertex3[1],
box.vertex4[1])
# collision test first
for line in img[miny:maxy, :, :]:
breakflag = False
for pixel in line[minx:maxx, :]: # !!! y -x !!!!!!!
if pixel[0] != 0:
breakflag = True
break
if breakflag:
collision_point = True
point_collide += 1
break
for vertex in existing_box_vertex:
breakflag = False
if minx < vertex[0] < maxx and miny < vertex[1] < maxy:
collision_box = False
box_collide += 1
break
else:
eb = existing_box_vertex.reshape(-1, 4, 2)
for ebox in eb:
maxxeb = max(ebox[0][0], ebox[1][0], ebox[2][0],
ebox[3][0])
minxeb = min(ebox[0][0], ebox[1][0], ebox[2][0],
ebox[3][0])
maxyeb = max(ebox[0][1], ebox[1][1], ebox[2][1],
ebox[3][1])
minyeb = min(ebox[0][1], ebox[1][1], ebox[2][1],
ebox[3][1])
if (minxeb < box.vertex1[0] < maxxeb and minyeb < box.vertex1[1] < maxyeb) or \
(minxeb < box.vertex2[0] < maxxeb and minyeb < box.vertex2[1] < maxyeb) or \
(minxeb < box.vertex3[0] < maxxeb and minyeb < box.vertex3[1] < maxyeb) or \
(minxeb < box.vertex4[0] < maxxeb and minyeb < box.vertex4[1] < maxyeb):
breakflag = True
break
if breakflag:
collision_box = True
box_collide += 1
break
# if collision_box or collision_point: # for testing
# for dot in point_sampled:
# if dot[1] < 30 and 100 / 6 > dot[0] > -100 / 6: # in range
# x, y = dot[1] * 180 / 30 + 20, dot[0] * 6 + 100
# x = int(x * resolution / 200)
# y = int(y * resolution / 200)
# enhanced = [[x, y] + e for e in pixel_enhance]
# for e in enhanced:
# if e[0] < resolution and 0 <= e[0] and e[1] < resolution and 0 <= e[0]:
# if collision_point and not collision_box:
# pp = torch.FloatTensor([1, 0, 0])
# if collision_box and not collision_point:
# pp = torch.FloatTensor([1, 1, 0])
# if collision_point and collision_box:
# pp = torch.FloatTensor([1, 1, 1])
# img[e[0], e[1]] = pp
# anninfo = {
# 'segmentation': [],
# 'area': box.length * box.width,
# 'image_id': anno[0]['image_id'],
# 'bbox': [box.xtl, box.ytl, box.width, box.length],
# 'rotation': box.alpha,
# 'category_id': 1,
# 'id': 999,
# 'iscrowd': 0
# }
# anno.append(anninfo)
#
# # box.resize(1 / 1.2)
# box.bBoxCalcVertxex()
# new_box_vertex = np.array([box.vertex1, box.vertex2, box.vertex3, box.vertex4]).reshape(-1, 2)
# existing_box_vertex = np.concatenate((existing_box_vertex, new_box_vertex))
if not (collision_box or collision_point):
for dot in point_sampled:
if dot[1] < 30 and 100 / 6 > dot[0] > -100 / 6: # in range
x, y = dot[1] * 900 / 30 + 100, dot[0] * 30 + 500
x = int(x)
y = int(y)
enhanced = [[x, y] + e for e in pixel_enhance]
for e in enhanced:
if e[0] < resolution and 0 <= e[0] and e[
1] < resolution and 0 <= e[0]:
pp = torch.FloatTensor([
# int(255 - math.hypot(dot[1], dot[0]) * 255 / 60) / 255,
# int(255 - (dot[1] * 235 / 30 + 20)) / 255,
# int(dot[0] * 75 / 15 + 80) / 255
# 0.4235, 0.9294, 1.0000
1,
1,
1
# ????????? how to calculate this RGB color encoding values ????
])
img[e[0], e[1]] = pp
anninfo = {
'segmentation': [],
'area': box.length * box.width,
'image_id': anno[0]['image_id'],
'bbox': [box.xtl, box.ytl, box.width, box.length],
'rotation': box.alpha,
'category_id': 1,
'id': 999,
'iscrowd': 0
}
anno.append(anninfo)
# box.resize(1 / 1.2)
box.bBoxCalcVertxex()
new_box_vertex = np.array(
[box.vertex1, box.vertex2, box.vertex3,
box.vertex4]).reshape(-1, 2)
existing_box_vertex = np.concatenate(
(existing_box_vertex, new_box_vertex))
img = img.permute(2, 0, 1)
img = transforms.ToPILImage()(img)
# print(point_collide, box_collide)
return img, anno
def __getitem__(self, idx):
img, anno = super(COCODataset, self).__getitem__(idx)
# img, anno = overlay_GT_on_scan(img, anno, self.gtcloud, self.gtann, resolution=1000)
noiseoffset = (torch.randn(2)) # minimal bbox noise is better?
for ann in anno:
noiseratio = ((torch.randn(1)).div_(20)).exp_().clamp(0.9, 1.1)
noiserotate = torch.randn(1).clamp(-3, 3)
label = ann["bbox"]
orien = ann["rotation"]
box = bBox_2D(
label[3], label[2], label[0] + label[2] / 2,
label[1] + label[3] / 2, orien
) # bBox_2D: length, width, xc, yc,alpha label: 'bbox': [box.xtl, box.ytl, box.width, box.length],
box.rotate(noiserotate)
box.resize(noiseratio)
# box.translate(noiseoffset[0], noiseoffset[1])
box.xcyc2topleft()
ann["bbox"] = [box.xtl, box.ytl, box.width, box.length]
# slightly stretch the box may be better viewed ?
ann["rotation"] = box.alpha
# filter crowd annotations
# TODO might be better to add an extra field
anno = [
obj for obj in anno
] # if obj["iscrowd"] == 0] ===============================================
boxes = [obj["bbox"] for obj in anno]
boxes = torch.as_tensor(boxes).reshape(-1, 4) # guard against no boxes
# print(boxes)
target = BoxList(boxes, img.size, mode="xywh").convert(
"xyxy") # =====================================
# print(target.bbox,'============================')
classes = [obj["category_id"] for obj in anno]
classes = [self.json_category_id_to_contiguous_id[c] for c in classes]
classes = torch.tensor(classes)
target.add_field("labels", classes)
masks = [obj["segmentation"] for obj in anno]
masks = SegmentationMask(masks, img.size)
target.add_field("masks", masks)
# ====================================
rotations = [obj["rotation"] * math.pi / 180 for obj in anno]
# print(rotations,'====')
rotations = torch.tensor(rotations)
rotations = torch.stack(
(5 * torch.sin(rotations), 5 * torch.cos(rotations)))
# COMPLEX space *5 is radius of unit circle or weight
# rotations = torch.stack()
rotations = torch.transpose(rotations, dim0=0, dim1=-1) # N*2 shape
# print(rotations)
target.add_field("rotations", rotations)
# print(target.get_field('rotations'), '============ooo================')
# print(target,'============================================')
target = target.clip_to_image(remove_empty=False)
# print(len(target), '==================targetanno=================')
if self.transforms is not None:
img, target = self.transforms(img, target)
# print(img.size(),'=================%d=================='%idx)
# print(target.get_field('rotations'), '============================')
return img, target, idx
def overlay_boxes(image, predictions, anntype, xylimits, res):
"""
Adds the predicted boxes on top of the image
Arguments:
image (np.ndarray): an image as returned by OpenCV
predictions (BoxList): the result of the computation by the model.
It should contain the field `labels`.
"""
# labels = predictions.get_field("labels")
imgsrc = image.copy()
oriens = predictions.get_field("rotations")
if anntype == 'output':
scores = predictions.get_field('scores')
else:
scores = oriens
boxes = predictions.bbox
xclist = []
yclist = []
alphalist = []
detections_per_frame = []
# print('\noriens:',oriens.size(),'boxes:',boxes.size(),'==========\n')
for box, orien, score in zip(boxes, oriens, scores):
color = {'targets': (155, 255, 255), 'output': (155, 255, 55)}
offset = {'targets': 2, 'output': 0}
box = box.squeeze_().detach().cpu().numpy()
box_ori = box.copy()
alpha = torch.atan2(orien[:][0], orien[:][1]) * 180 / math.pi
# alpha = (orien[:][0] + orien[:][1]) * 0.5 * 180 / math.pi # for testing!
alpha = alpha.squeeze_().detach().cpu().numpy()
# print(alpha,anntype,'====')
if anntype == 'output':
score.squeeze_().detach().cpu().numpy()
# top_left, bottom_right = box[:2].tolist(), box[2:].tolist()
top_left, bottom_right = box[:2], box[2:]
l = bottom_right[1] - top_left[1]
w = bottom_right[0] - top_left[0]
xc = (top_left[0] + bottom_right[0]) / 2
yc = (top_left[1] + bottom_right[1]) / 2
# if not (xylimits[0] < xc < xylimits[1] and xylimits[2] < yc < xylimits[3]):
# # continue
# pass
# cv2.line(image, (xylimits[0], xylimits[3]), (xylimits[1], xylimits[3]), color[anntype], 1, cv2.LINE_AA)
# cv2.line(image, (xylimits[0], xylimits[2]), (xylimits[1], xylimits[2]), color[anntype], 1, cv2.LINE_AA)
# cv2.line(image, (xylimits[0], xylimits[3]), (xylimits[0], xylimits[2]), color[anntype], 1, cv2.LINE_AA)
# cv2.line(image, (xylimits[1], xylimits[3]), (xylimits[1], xylimits[2]), color[anntype], 1, cv2.LINE_AA)
xclist.append(xc)
yclist.append(yc)
alphalist.append(alpha)
# if l * w <= 1:
# continue
box = bBox_2D(l, w, xc + offset[anntype], yc + offset[anntype], alpha)
box.scale(res / 600, 0, 0)
# box.resize(1.2)
box.bBoxCalcVertxex()
rad = box.alpha * math.pi / 180
cv2.line(image, box.vertex1, box.vertex2, color[anntype], 2, cv2.LINE_AA)
cv2.line(image, box.vertex2, box.vertex4, color[anntype], 2, cv2.LINE_AA)
cv2.line(image, box.vertex3, box.vertex1, color[anntype], 2, cv2.LINE_AA)
cv2.line(image, box.vertex4, box.vertex3, color[anntype], 2, cv2.LINE_AA)
if anntype == 'output':
print('+++++')
print(box.vertex4, box.vertex3, box.vertex2, box.vertex1, '====', l * w, '\t', l, '\t', w, '\t angle',
box.alpha, ' score ', score)
detections_per_frame.append([score, (box.yc - 100) / 30.0, (box.xc - 500) / 30.0, rad])
else:
# print(box.vertex4, box.vertex3, box.vertex2, box.vertex1, '====', l * w, '\t', l, '\t', w, '\t angle',
# box.alpha)
pass
point = int(box.xc - box.length * 0.8 * np.sin(rad)), int(box.yc + box.length * 0.8 * np.cos(rad))
cv2.line(image, (int(box.xc), int(box.yc)),
point,
color[anntype], 2, cv2.LINE_AA)
if anntype == 'output':
cv2.putText(image, str(score.numpy()), point, fontFace=1, fontScale=1.5, color=(255, 0, 255))
image = cv2.addWeighted(imgsrc, 0.4, image, 0.6, 0)
if anntype == 'output':
detections_per_frame = np.array(detections_per_frame)
else:
detections_per_frame = []
return np.array([xclist, yclist], dtype=float), np.array(alphalist, dtype=float), detections_per_frame