def to_rotated_boxes(boxes, oriens): boxes_with_oriens = torch.cat((boxes, oriens), dim=-1) # print(boxes) for j, boxnorien in enumerate(boxes_with_oriens): # boxnorien = boxnorien.squeeze_() alpha = torch.atan2(boxnorien[:][-2], boxnorien[:][-1]) * 180 / 3.1415926 # alpha = alpha.squeeze_() # print(alpha,anntype,'====') # top_left, bottom_right = box[:2].tolist(), box[2:].tolist() top_left, bottom_right = boxnorien[:2], boxnorien[2:4] l = bottom_right[1] - top_left[1] w = bottom_right[0] - top_left[0] xc = (top_left[0] + bottom_right[0]) / 2 yc = (top_left[1] + bottom_right[1]) / 2 # print(alpha,top_left,bottom_right,'=====') if l * w <= 1: continue box = bBox_2D(l, w, xc, yc, alpha) box.bBoxCalcVertxex() print(boxes.size(), oriens.size(), boxnorien.size()) # boxes[j]=torch. return
def overlay_boxes(image, predictions, anntype): """ Adds the predicted boxes on top of the image Arguments: image (np.ndarray): an image as returned by OpenCV predictions (BoxList): the result of the computation by the model. It should contain the field `labels`. """ # labels = predictions.get_field("labels") oriens = predictions.get_field("rotations") boxes = predictions.bbox xclist = [] yclist = [] alphalist = [] # print('\noriens:',oriens.size(),'boxes:',boxes.size(),'==========\n') for box, orien in zip(boxes, oriens): color = {'targets': (155, 255, 255), 'output': (155, 255, 55)} offset = {'targets': 2, 'output': 0} box = box.squeeze_().detach().cpu().numpy() alpha = torch.atan2(orien[:][0], orien[:][1]) * 180 / math.pi alpha = alpha.squeeze_().detach().cpu().numpy() # print(alpha,anntype,'====') # top_left, bottom_right = box[:2].tolist(), box[2:].tolist() top_left, bottom_right = box[:2], box[2:] l = bottom_right[1] - top_left[1] w = bottom_right[0] - top_left[0] xc = (top_left[0] + bottom_right[0]) / 2 yc = (top_left[1] + bottom_right[1]) / 2 xclist.append(xc) yclist.append(yc) alphalist.append(alpha) # if l * w <= 1: # continue box = bBox_2D(l, w, xc + offset[anntype], yc + offset[anntype], alpha) box.bBoxCalcVertxex() rad = box.alpha * math.pi / 180 cv2.line(image, box.vertex1, box.vertex2, color[anntype], 1, cv2.LINE_AA) cv2.line(image, box.vertex2, box.vertex4, color[anntype], 1, cv2.LINE_AA) cv2.line(image, box.vertex3, box.vertex1, color[anntype], 1, cv2.LINE_AA) cv2.line(image, box.vertex4, box.vertex3, color[anntype], 1, cv2.LINE_AA) # print(box.vertex4, box.vertex3, box.vertex2, box.vertex1, '====',l*w,'\t',l,'\t',w) point = int(box.xc - box.length * 0.8 * np.sin(rad)), int(box.yc + box.length * 0.8 * np.cos(rad)) cv2.line(image, (int(box.xc), int(box.yc)), point, color[anntype], 1, cv2.LINE_AA) return np.array([xclist, yclist], dtype=float), np.array(alphalist, dtype=float)
def overlay_GT_on_scan(img, anno, cloudgt, anngt, resolution=1000): # one img and its ann (ann in pixels) # GT database of cloudgt and anngt (cloud and ann in original METERs !!!) # thus, box.resize is needed point_collide = 0 box_collide = 0 if len(anno) == 0: return img, anno ann_num = len(anno) sampling_num = int(max( 0, (15 - ann_num))) # 12 is the max box num in this dataset # if ann_num <= 5: # sampling_num = int(random.random() * max(0, (15 - ann_num))) # 12 is the max box num in this dataset # else: # return img, anno collision_box = False # indicator for collision test collision_point = False _pixel_enhance = np.array([-1, 0, 1]) pixel_enhance = np.array([[x, y] for x in _pixel_enhance for y in _pixel_enhance ]) # enhance pixel by extra 8 existing_box_vertex = [] for ann in anno: # for every existing GT box label = ann["bbox"] orien = ann["rotation"] ebox = bBox_2D(label[3], label[2], label[0] + label[2] / 2, label[1] + label[3] / 2, orien) # bBox_2D: length, width, xc, yc,alpha label: 'bbox': [box.xtl, box.ytl, box.width, box.length], # ebox.resize(1.2) ebox.bBoxCalcVertxex() existing_box_vertex.append( [ebox.vertex1, ebox.vertex2, ebox.vertex3, ebox.vertex4]) # shape [N,4,2] existing_box_vertex = np.array(existing_box_vertex).reshape(-1, 2) img = transforms.ToTensor()(img) img = img.permute(1, 2, 0) for num in range(sampling_num): # for every sampled GT box index = int(random.random() * len(anngt)) ann_sampled = anngt[index] point_sampled = cloudgt[index] box = bBox_2D(ann_sampled['bbox'][3], ann_sampled['bbox'][2], ann_sampled['bbox'][0], ann_sampled['bbox'][1], ann_sampled['rotation']) box.scale(900 / 30, 500, 100) # box.scale(resolution / 200, 0, 0) # converse to img pixel values # box.resize(1.2) # to ensure points inside box.bBoxCalcVertxex() box.xcyc2topleft() maxx = max(box.vertex1[0], box.vertex2[0], box.vertex3[0], box.vertex4[0]) minx = min(box.vertex1[0], box.vertex2[0], box.vertex3[0], box.vertex4[0]) maxy = max(box.vertex1[1], box.vertex2[1], box.vertex3[1], box.vertex4[1]) miny = min(box.vertex1[1], box.vertex2[1], box.vertex3[1], box.vertex4[1]) # collision test first for line in img[miny:maxy, :, :]: breakflag = False for pixel in line[minx:maxx, :]: # !!! y -x !!!!!!! if pixel[0] != 0: breakflag = True break if breakflag: collision_point = True point_collide += 1 break for vertex in existing_box_vertex: breakflag = False if minx < vertex[0] < maxx and miny < vertex[1] < maxy: collision_box = False box_collide += 1 break else: eb = existing_box_vertex.reshape(-1, 4, 2) for ebox in eb: maxxeb = max(ebox[0][0], ebox[1][0], ebox[2][0], ebox[3][0]) minxeb = min(ebox[0][0], ebox[1][0], ebox[2][0], ebox[3][0]) maxyeb = max(ebox[0][1], ebox[1][1], ebox[2][1], ebox[3][1]) minyeb = min(ebox[0][1], ebox[1][1], ebox[2][1], ebox[3][1]) if (minxeb < box.vertex1[0] < maxxeb and minyeb < box.vertex1[1] < maxyeb) or \ (minxeb < box.vertex2[0] < maxxeb and minyeb < box.vertex2[1] < maxyeb) or \ (minxeb < box.vertex3[0] < maxxeb and minyeb < box.vertex3[1] < maxyeb) or \ (minxeb < box.vertex4[0] < maxxeb and minyeb < box.vertex4[1] < maxyeb): breakflag = True break if breakflag: collision_box = True box_collide += 1 break # if collision_box or collision_point: # for testing # for dot in point_sampled: # if dot[1] < 30 and 100 / 6 > dot[0] > -100 / 6: # in range # x, y = dot[1] * 180 / 30 + 20, dot[0] * 6 + 100 # x = int(x * resolution / 200) # y = int(y * resolution / 200) # enhanced = [[x, y] + e for e in pixel_enhance] # for e in enhanced: # if e[0] < resolution and 0 <= e[0] and e[1] < resolution and 0 <= e[0]: # if collision_point and not collision_box: # pp = torch.FloatTensor([1, 0, 0]) # if collision_box and not collision_point: # pp = torch.FloatTensor([1, 1, 0]) # if collision_point and collision_box: # pp = torch.FloatTensor([1, 1, 1]) # img[e[0], e[1]] = pp # anninfo = { # 'segmentation': [], # 'area': box.length * box.width, # 'image_id': anno[0]['image_id'], # 'bbox': [box.xtl, box.ytl, box.width, box.length], # 'rotation': box.alpha, # 'category_id': 1, # 'id': 999, # 'iscrowd': 0 # } # anno.append(anninfo) # # # box.resize(1 / 1.2) # box.bBoxCalcVertxex() # new_box_vertex = np.array([box.vertex1, box.vertex2, box.vertex3, box.vertex4]).reshape(-1, 2) # existing_box_vertex = np.concatenate((existing_box_vertex, new_box_vertex)) if not (collision_box or collision_point): for dot in point_sampled: if dot[1] < 30 and 100 / 6 > dot[0] > -100 / 6: # in range x, y = dot[1] * 900 / 30 + 100, dot[0] * 30 + 500 x = int(x) y = int(y) enhanced = [[x, y] + e for e in pixel_enhance] for e in enhanced: if e[0] < resolution and 0 <= e[0] and e[ 1] < resolution and 0 <= e[0]: pp = torch.FloatTensor([ # int(255 - math.hypot(dot[1], dot[0]) * 255 / 60) / 255, # int(255 - (dot[1] * 235 / 30 + 20)) / 255, # int(dot[0] * 75 / 15 + 80) / 255 # 0.4235, 0.9294, 1.0000 1, 1, 1 # ????????? how to calculate this RGB color encoding values ???? ]) img[e[0], e[1]] = pp anninfo = { 'segmentation': [], 'area': box.length * box.width, 'image_id': anno[0]['image_id'], 'bbox': [box.xtl, box.ytl, box.width, box.length], 'rotation': box.alpha, 'category_id': 1, 'id': 999, 'iscrowd': 0 } anno.append(anninfo) # box.resize(1 / 1.2) box.bBoxCalcVertxex() new_box_vertex = np.array( [box.vertex1, box.vertex2, box.vertex3, box.vertex4]).reshape(-1, 2) existing_box_vertex = np.concatenate( (existing_box_vertex, new_box_vertex)) img = img.permute(2, 0, 1) img = transforms.ToPILImage()(img) # print(point_collide, box_collide) return img, anno
def __getitem__(self, idx): img, anno = super(COCODataset, self).__getitem__(idx) # img, anno = overlay_GT_on_scan(img, anno, self.gtcloud, self.gtann, resolution=1000) noiseoffset = (torch.randn(2)) # minimal bbox noise is better? for ann in anno: noiseratio = ((torch.randn(1)).div_(20)).exp_().clamp(0.9, 1.1) noiserotate = torch.randn(1).clamp(-3, 3) label = ann["bbox"] orien = ann["rotation"] box = bBox_2D( label[3], label[2], label[0] + label[2] / 2, label[1] + label[3] / 2, orien ) # bBox_2D: length, width, xc, yc,alpha label: 'bbox': [box.xtl, box.ytl, box.width, box.length], box.rotate(noiserotate) box.resize(noiseratio) # box.translate(noiseoffset[0], noiseoffset[1]) box.xcyc2topleft() ann["bbox"] = [box.xtl, box.ytl, box.width, box.length] # slightly stretch the box may be better viewed ? ann["rotation"] = box.alpha # filter crowd annotations # TODO might be better to add an extra field anno = [ obj for obj in anno ] # if obj["iscrowd"] == 0] =============================================== boxes = [obj["bbox"] for obj in anno] boxes = torch.as_tensor(boxes).reshape(-1, 4) # guard against no boxes # print(boxes) target = BoxList(boxes, img.size, mode="xywh").convert( "xyxy") # ===================================== # print(target.bbox,'============================') classes = [obj["category_id"] for obj in anno] classes = [self.json_category_id_to_contiguous_id[c] for c in classes] classes = torch.tensor(classes) target.add_field("labels", classes) masks = [obj["segmentation"] for obj in anno] masks = SegmentationMask(masks, img.size) target.add_field("masks", masks) # ==================================== rotations = [obj["rotation"] * math.pi / 180 for obj in anno] # print(rotations,'====') rotations = torch.tensor(rotations) rotations = torch.stack( (5 * torch.sin(rotations), 5 * torch.cos(rotations))) # COMPLEX space *5 is radius of unit circle or weight # rotations = torch.stack() rotations = torch.transpose(rotations, dim0=0, dim1=-1) # N*2 shape # print(rotations) target.add_field("rotations", rotations) # print(target.get_field('rotations'), '============ooo================') # print(target,'============================================') target = target.clip_to_image(remove_empty=False) # print(len(target), '==================targetanno=================') if self.transforms is not None: img, target = self.transforms(img, target) # print(img.size(),'=================%d=================='%idx) # print(target.get_field('rotations'), '============================') return img, target, idx
def overlay_boxes(image, predictions, anntype, xylimits, res): """ Adds the predicted boxes on top of the image Arguments: image (np.ndarray): an image as returned by OpenCV predictions (BoxList): the result of the computation by the model. It should contain the field `labels`. """ # labels = predictions.get_field("labels") imgsrc = image.copy() oriens = predictions.get_field("rotations") if anntype == 'output': scores = predictions.get_field('scores') else: scores = oriens boxes = predictions.bbox xclist = [] yclist = [] alphalist = [] detections_per_frame = [] # print('\noriens:',oriens.size(),'boxes:',boxes.size(),'==========\n') for box, orien, score in zip(boxes, oriens, scores): color = {'targets': (155, 255, 255), 'output': (155, 255, 55)} offset = {'targets': 2, 'output': 0} box = box.squeeze_().detach().cpu().numpy() box_ori = box.copy() alpha = torch.atan2(orien[:][0], orien[:][1]) * 180 / math.pi # alpha = (orien[:][0] + orien[:][1]) * 0.5 * 180 / math.pi # for testing! alpha = alpha.squeeze_().detach().cpu().numpy() # print(alpha,anntype,'====') if anntype == 'output': score.squeeze_().detach().cpu().numpy() # top_left, bottom_right = box[:2].tolist(), box[2:].tolist() top_left, bottom_right = box[:2], box[2:] l = bottom_right[1] - top_left[1] w = bottom_right[0] - top_left[0] xc = (top_left[0] + bottom_right[0]) / 2 yc = (top_left[1] + bottom_right[1]) / 2 # if not (xylimits[0] < xc < xylimits[1] and xylimits[2] < yc < xylimits[3]): # # continue # pass # cv2.line(image, (xylimits[0], xylimits[3]), (xylimits[1], xylimits[3]), color[anntype], 1, cv2.LINE_AA) # cv2.line(image, (xylimits[0], xylimits[2]), (xylimits[1], xylimits[2]), color[anntype], 1, cv2.LINE_AA) # cv2.line(image, (xylimits[0], xylimits[3]), (xylimits[0], xylimits[2]), color[anntype], 1, cv2.LINE_AA) # cv2.line(image, (xylimits[1], xylimits[3]), (xylimits[1], xylimits[2]), color[anntype], 1, cv2.LINE_AA) xclist.append(xc) yclist.append(yc) alphalist.append(alpha) # if l * w <= 1: # continue box = bBox_2D(l, w, xc + offset[anntype], yc + offset[anntype], alpha) box.scale(res / 600, 0, 0) # box.resize(1.2) box.bBoxCalcVertxex() rad = box.alpha * math.pi / 180 cv2.line(image, box.vertex1, box.vertex2, color[anntype], 2, cv2.LINE_AA) cv2.line(image, box.vertex2, box.vertex4, color[anntype], 2, cv2.LINE_AA) cv2.line(image, box.vertex3, box.vertex1, color[anntype], 2, cv2.LINE_AA) cv2.line(image, box.vertex4, box.vertex3, color[anntype], 2, cv2.LINE_AA) if anntype == 'output': print('+++++') print(box.vertex4, box.vertex3, box.vertex2, box.vertex1, '====', l * w, '\t', l, '\t', w, '\t angle', box.alpha, ' score ', score) detections_per_frame.append([score, (box.yc - 100) / 30.0, (box.xc - 500) / 30.0, rad]) else: # print(box.vertex4, box.vertex3, box.vertex2, box.vertex1, '====', l * w, '\t', l, '\t', w, '\t angle', # box.alpha) pass point = int(box.xc - box.length * 0.8 * np.sin(rad)), int(box.yc + box.length * 0.8 * np.cos(rad)) cv2.line(image, (int(box.xc), int(box.yc)), point, color[anntype], 2, cv2.LINE_AA) if anntype == 'output': cv2.putText(image, str(score.numpy()), point, fontFace=1, fontScale=1.5, color=(255, 0, 255)) image = cv2.addWeighted(imgsrc, 0.4, image, 0.6, 0) if anntype == 'output': detections_per_frame = np.array(detections_per_frame) else: detections_per_frame = [] return np.array([xclist, yclist], dtype=float), np.array(alphalist, dtype=float), detections_per_frame