def generate_all_anchors(self, im_c, size): if self.image_center == im_c and self.size == size: return False self.image_center = im_c self.size = size a0x = im_c - size // 2 * self.stride ori = np.array([a0x] * 4, dtype=np.float32) zero_anchors = self.anchors + ori x1 = zero_anchors[:, 0] y1 = zero_anchors[:, 1] x2 = zero_anchors[:, 2] y2 = zero_anchors[:, 3] x1, y1, x2, y2 = map(lambda x: x.reshape(self.anchor_num, 1, 1), [x1, y1, x2, y2]) cx, cy, w, h = corner2center([x1, y1, x2, y2]) disp_x = np.arange(0, size).reshape(1, 1, -1) * self.stride disp_y = np.arange(0, size).reshape(1, -1, 1) * self.stride cx = cx + disp_x cy = cy + disp_y # broadcast zero = np.zeros((self.anchor_num, size, size), dtype=np.float32) cx, cy, w, h = map(lambda x: x + zero, [cx, cy, w, h]) x1, y1, x2, y2 = center2corner([cx, cy, w, h]) self.all_anchors = np.stack([x1, y1, x2, y2]), np.stack([cx, cy, w, h]) return True
def crop(self, inp): # Case for image input. if inp.shape == torch.Size([self.imgHeight, self.imgWidth, 3]): image = inp # Return 300x300 patch if no object is detected. if self.locations is None: return image[0:300, 0:300, :] # Check the ios of the cropped image with oracle bounding box to ensure at least one labeled item. found = False cnt = 0 while not found: cnt += 1 if cnt > 300: self.crop_coordinates = torch.Tensor([150, 0, 450, 300]) image = image[int(self.crop_coordinates[1] ):int(self.crop_coordinates[3]), int(self.crop_coordinates[0] ):int(self.crop_coordinates[2]), :] break crop = random.randint(0, self.imgWidth - 300) self.crop_coordinates = torch.Tensor( [crop, 0, crop + 300, 300]) for location in self.locations: if helper.ios(location, helper.corner2center(self.crop_coordinates) ) > self.cropping_ios_threshold: found = True image = image[int(self.crop_coordinates[1] ):int(self.crop_coordinates[3]), int(self.crop_coordinates[0] ):int(self.crop_coordinates[2]), :] break return image # Case for location input. locations = inp locations[:, 0] -= self.crop_coordinates[0] # Set locations to 0 if the ios is too small. ios = helper.ios(locations, torch.Tensor([150, 150, 300, 300])) self.ios_index = ios > self.cropping_ios_threshold locations[ios <= self.cropping_ios_threshold] = 0 locations = locations[self.ios_index] # Clip the location. locations = helper.center2corner(locations) locations = torch.clamp(locations, 0, 300) locations = helper.corner2center(locations) # Save the oracle locations. self.locations = locations return locations
def test_corner2(self): prior_layer_cfg = [{ 'layer_name': 'Conv5', 'feature_dim_hw': (19, 19), 'bbox_size': (60, 60), 'aspect_ratio': (1.0, 1 / 2, 1 / 3, 2.0, 3.0, '1t') }, { 'layer_name': 'Conv11', 'feature_dim_hw': (10, 10), 'bbox_size': (105, 105), 'aspect_ratio': (1.0, 1 / 2, 1 / 3, 2.0, 3.0, '1t') }, { 'layer_name': 'Conv14_2', 'feature_dim_hw': (5, 5), 'bbox_size': (150, 150), 'aspect_ratio': (1.0, 1 / 2, 1 / 3, 2.0, 3.0, '1t') }, { 'layer_name': 'Conv15_2', 'feature_dim_hw': (3, 3), 'bbox_size': (195, 195), 'aspect_ratio': (1.0, 1 / 2, 1 / 3, 2.0, 3.0, '1t') }, { 'layer_name': 'Conv16_2', 'feature_dim_hw': (2, 2), 'bbox_size': (240, 240), 'aspect_ratio': (1.0, 1 / 2, 1 / 3, 2.0, 3.0, '1t') }, { 'layer_name': 'Conv17_2', 'feature_dim_hw': (1, 1), 'bbox_size': (285, 285), 'aspect_ratio': (1.0, 1 / 2, 1 / 3, 2.0, 3.0, '1t') }] pp = generate_prior_bboxes(prior_layer_cfg) print('original', pp[0]) test = center2corner(pp[0]) print('corner', test) test = corner2center(test) print('center', test) torch.set_default_tensor_type('torch.cuda.FloatTensor') print('Pytorch CUDA Enabled?:', torch.cuda.is_available()) b = 0.5 * torch.eye(3) b_gpu = b.cuda() print(b_gpu)
test_img = np.array(test_img) test_img = np.subtract(test_img, [127, 127, 127]) test_img = np.divide(test_img, 128) test_img = test_img.reshape( (test_img.shape[2], test_img.shape[0], test_img.shape[1])) test_img = torch.Tensor(test_img) test_img = test_img.unsqueeze(0).cpu() images = Variable(test_img) # Use Variable(*) to allow gradient flow\n", conf_preds, loc_preds = net.forward(images) # Forward once\n", print(conf_preds.shape) print(loc_preds.shape) bbox = bbox_helper.loc2bbox(loc_preds, prior_bboxes) bbox = bbox[0].detach() bbox_corner = bbox_helper.center2corner(bbox) print(bbox_corner) print(conf_preds) print(bbox_corner.shape) bbox_corner = bbox_corner conf_preds = conf_preds[0].detach() # idx = conf_preds[:, 2] > 0.6 # bbox_corner = bbox_corner[idx] # bbox = bbox[idx] # print(bbox_corner) bbox_nms = bbox_helper.nms_bbox(bbox_corner, conf_preds) print(bbox_nms[0]) bbox_nms = torch.Tensor(bbox_nms[0]) print(bbox_nms.shape)
def preview(self, index=0): if index == -1: index = random.randint(0, len(self.dataset_list)) # Acquire preview targets. image, confidences, locations = self[index] image = image.view(image.shape[1], image.shape[2], image.shape[0]) # Denormalize the data. image = self.denormalize(image) locations = self.denormalize(locations) # Prepare data for plotting. image = np.array(image).astype(int) fig, ax = plt.subplots(1) ax.imshow(image) matched_rect, oracle_rect = None, None # Display matched bounding boxes. for i_location in range(0, locations.shape[0]): if confidences[i_location] != 0: corner = helper.center2corner(locations[i_location]) x = corner[0] y = corner[1] matched_rect = patches.Rectangle((x, y), locations[i_location][2], locations[i_location][3], linewidth=1, edgecolor='r', facecolor='none') ax.add_patch(matched_rect) # Display ground truth bounding boxes. for i_location in range(0, self.locations.shape[0]): corner = helper.center2corner(self.locations[i_location]) x = corner[0] y = corner[1] oracle_rect = patches.Rectangle((x, y), self.locations[i_location][2], self.locations[i_location][3], linewidth=1, edgecolor='g', facecolor='none') ax.add_patch(oracle_rect) fig.canvas.set_window_title('Preview at Index [' + str(index) + ']') plt.title('Preview at Index [' + str(index) + ']') plt.xlim(0, 300) plt.ylim(300, 0) if matched_rect is not None: plt.legend([oracle_rect, matched_rect], ['Oracle Box', 'Matched Box'], bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.) else: plt.legend([oracle_rect], ['Oracle Box'], bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.) plt.tight_layout() # Output stats. print('Available classes:', self.classes) print('Number of oracle bounding boxes:', self.locations.shape[0]) print('Number of matched bounding boxes:', locations.shape[0]) print('Bounding box configuration:') print(np.array(self.prior_layer_cfg)) plt.show()
def test_random2(self): torch.set_default_tensor_type('torch.cuda.FloatTensor') torch.set_printoptions(precision=10) prior_layer_cfg = [{ 'layer_name': 'Conv5', 'feature_dim_hw': (19, 19), 'bbox_size': (60, 60), 'aspect_ratio': (1.0, 1 / 2, 1 / 3, 2.0, 3.0, '1t') }, { 'layer_name': 'Conv11', 'feature_dim_hw': (10, 10), 'bbox_size': (105, 105), 'aspect_ratio': (1.0, 1 / 2, 1 / 3, 2.0, 3.0, '1t') }, { 'layer_name': 'Conv14_2', 'feature_dim_hw': (5, 5), 'bbox_size': (150, 150), 'aspect_ratio': (1.0, 1 / 2, 1 / 3, 2.0, 3.0, '1t') }, { 'layer_name': 'Conv15_2', 'feature_dim_hw': (3, 3), 'bbox_size': (195, 195), 'aspect_ratio': (1.0, 1 / 2, 1 / 3, 2.0, 3.0, '1t') }, { 'layer_name': 'Conv16_2', 'feature_dim_hw': (2, 2), 'bbox_size': (240, 240), 'aspect_ratio': (1.0, 1 / 2, 1 / 3, 2.0, 3.0, '1t') }, { 'layer_name': 'Conv17_2', 'feature_dim_hw': (1, 1), 'bbox_size': (285, 285), 'aspect_ratio': (1.0, 1 / 2, 1 / 3, 2.0, 3.0, '1t') }] pp = generate_prior_bboxes(prior_layer_cfg) # test_list = load_data('../Debugimage', '../Debuglabel') test_list = load_data('../cityscapes_samples', '../cityscapes_samples_labels') #print(test_list) test_dataset = CityScapeDataset(test_list) test_data_loader = torch.utils.data.DataLoader(test_dataset, batch_size=1, shuffle=True, num_workers=0) lfw_dataset_dir = '../' test_net = ssd_net.SSD(3) test_net_state = torch.load( os.path.join(lfw_dataset_dir, 'ssd_net.pth')) test_net.load_state_dict(test_net_state) idx, (img, bbox, label) = next(enumerate(test_data_loader)) pred_cof, pred_loc = test_net.forward(img) print(pred_loc.shape) import torch.nn.functional as F pred_loc = pred_loc.detach() bbox_center = loc2bbox(pred_loc[0], pp) pred_cof = F.softmax(pred_cof[0]) ind = np.where(pred_cof > 0.7) # pred_cof = F.softmax(pred_cof[ind[0]]) bbox_center = bbox_center[ind[0]] print(ind, pred_cof) img = img[0].cpu().numpy() img = img.reshape((300, 300, 3)) img = (img * 128 + np.asarray([[127, 127, 127]])) / 255 fig, ax = plt.subplots(1) imageB_array = resize(img, (600, 1200), anti_aliasing=True) ax.imshow(imageB_array, cmap='brg') bbox_corner = center2corner(bbox_center) for i in range(0, bbox_corner.shape[0]): # print('i point', bbox_corner[i, 0]*600, bbox_corner[i, 1]*300,(bbox_corner[i, 2]-bbox_corner[i, 0])*600, (bbox_corner[i, 3]-bbox_corner[i, 1])*300) rect = patches.Rectangle( (bbox_corner[i, 0] * 1200, bbox_corner[i, 1] * 600), (bbox_corner[i, 2] - bbox_corner[i, 0]) * 1200, (bbox_corner[i, 3] - bbox_corner[i, 1]) * 600, linewidth=2, edgecolor='r', facecolor='none') # Create a Rectangle patch ax.add_patch(rect) # Add the patch to the Axes plt.show()
def test_dataLoad(self): torch.set_default_tensor_type('torch.cuda.FloatTensor') torch.set_printoptions(precision=10) prior_layer_cfg = [{ 'layer_name': 'Conv5', 'feature_dim_hw': (19, 19), 'bbox_size': (60, 60), 'aspect_ratio': (1.0, 1 / 2, 1 / 3, 2.0, 3.0, '1t') }, { 'layer_name': 'Conv11', 'feature_dim_hw': (10, 10), 'bbox_size': (105, 105), 'aspect_ratio': (1.0, 1 / 2, 1 / 3, 2.0, 3.0, '1t') }, { 'layer_name': 'Conv14_2', 'feature_dim_hw': (5, 5), 'bbox_size': (150, 150), 'aspect_ratio': (1.0, 1 / 2, 1 / 3, 2.0, 3.0, '1t') }, { 'layer_name': 'Conv15_2', 'feature_dim_hw': (3, 3), 'bbox_size': (195, 195), 'aspect_ratio': (1.0, 1 / 2, 1 / 3, 2.0, 3.0, '1t') }, { 'layer_name': 'Conv16_2', 'feature_dim_hw': (2, 2), 'bbox_size': (240, 240), 'aspect_ratio': (1.0, 1 / 2, 1 / 3, 2.0, 3.0, '1t') }, { 'layer_name': 'Conv17_2', 'feature_dim_hw': (1, 1), 'bbox_size': (285, 285), 'aspect_ratio': (1.0, 1 / 2, 1 / 3, 2.0, 3.0, '1t') }] pp = generate_prior_bboxes(prior_layer_cfg) #test_list = load_data('../Debugimage', '../Debuglabel') test_list = load_data('../cityscapes_samples', '../cityscapes_samples_labels') print(test_list) gt_bbox = np.asarray(test_list[0]['label'][1]) * [ 300 / 2048, 300 / 1024, 300 / 2048, 300 / 1024 ] print('ground truth from file:', test_list[0]['label'][0]) test_dataset = CityScapeDataset(test_list) test_data_loader = torch.utils.data.DataLoader(test_dataset, batch_size=1, shuffle=True, num_workers=0) idx, (img, bbox, label) = next(enumerate(test_data_loader)) bbox = bbox[0] label = label[0] print(bbox.shape, label.shape) print('matched label', label[np.where(label > 0)], np.where(label > 0), label.shape) print('first bbox from data_set:', bbox[0], label[0]) bbox_center = loc2bbox(bbox, pp) bbox_corner = center2corner(bbox_center) img = img[0].cpu().numpy() img = img.reshape((300, 300, 3)) img = (img * 128 + np.asarray([[127, 127, 127]])) / 255 # for i in range(0, bbox.shape[0]): # cv2.rectangle(img, (bbox[i,0], bbox[i,1]), (bbox[i,2], bbox[i,3]), (0, 255, 0), 3) #cv2.imshow("img", img) # Create figure and axes fig, ax = plt.subplots(1) imageB_array = resize(img, (300, 300), anti_aliasing=True) ax.imshow(imageB_array, cmap='brg') bbox_corner = bbox_corner.cpu().numpy() bbox_corner = bbox_corner[np.where(label > 0)] temp_lab = label[np.where(label > 0)] print('matched bbox ======', bbox_corner) pp = center2corner(pp) pp = pp[np.where(label > 0)] print('864 tensor: ', pp) for i in range(0, bbox_corner.shape[0]): if temp_lab[i] == 1: # print('i point', bbox_corner[i, 0]*600, bbox_corner[i, 1]*300,(bbox_corner[i, 2]-bbox_corner[i, 0])*600, (bbox_corner[i, 3]-bbox_corner[i, 1])*300) rect = patches.Rectangle( (bbox_corner[i, 0] * 300, bbox_corner[i, 1] * 300), (bbox_corner[i, 2] - bbox_corner[i, 0]) * 300, (bbox_corner[i, 3] - bbox_corner[i, 1]) * 300, linewidth=2, edgecolor='r', facecolor='none') # Create a Rectangle patch ax.add_patch(rect) # Add the patch to the Axes else: rect = patches.Rectangle( (bbox_corner[i, 0] * 1200, bbox_corner[i, 1] * 600), (bbox_corner[i, 2] - bbox_corner[i, 0]) * 1200, (bbox_corner[i, 3] - bbox_corner[i, 1]) * 600, linewidth=2, edgecolor='y', facecolor='none') # Create a Rectangle patch ax.add_patch(rect) # Add the patch to the Axes for i in range(0, pp.shape[0]): rect = patches.Rectangle( (pp[i, 0] * 300, pp[i, 1] * 300), (pp[i, 2] - pp[i, 0]) * 300, (pp[i, 3] - pp[i, 1]) * 300, linewidth=1, edgecolor='blue', facecolor='none') # Create a Rectangle patch ax.add_patch(rect) # Add the patch to the Axes # for i in range(0, gt_bbox.shape[0]): # rect = patches.Rectangle((gt_bbox[i][0], gt_bbox[i][1]), # (gt_bbox[i][2] - gt_bbox[i][0]), # (gt_bbox[i][3] - gt_bbox[i][1]), linewidth=1, edgecolor='g', # facecolor='none') # Create a Rectangle patch # ax.add_patch(rect) # Add the patch to the Axes plt.show()
else: test_net_state = torch.load(path_to_trained_model, map_location='cpu') test_net = SSD(num_classes=3) test_net.load_state_dict(test_net_state) test_net.eval() test_image_permuted = img_tensor.permute(0, 3, 1, 2) test_image_permuted = Variable(test_image_permuted.float()) test_conf_preds, test_loc_preds = test_net.forward(test_image_permuted) test_bbox_priors = prior_bboxes.unsqueeze(0) test_bbox_preds = loc2bbox(test_loc_preds.cpu(), test_bbox_priors.cpu(), center_var=0.1, size_var=0.2) sel_bbox_preds = nms_bbox(test_bbox_preds.squeeze().detach(), test_conf_preds.squeeze().detach().cpu(), overlap_threshold=0.5, prob_threshold=0.5) rects = [] classes = [] for key in sel_bbox_preds.keys(): for value in sel_bbox_preds[key]: classes.append(key) rects.append(value) rects = center2corner(torch.tensor(rects))*300 if len(rects) != 0: if rects.dim() == 1: rects = rects.unsqueeze(0) classes = torch.tensor(classes) drawRectsWithImgPLT(img, rects, classes) else: print("no object detected in this img")
def main(): torch.set_default_tensor_type('torch.cuda.FloatTensor') prior_layer_cfg = [{ 'layer_name': 'Conv5', 'feature_dim_hw': (19, 19), 'bbox_size': (60, 60), 'aspect_ratio': (1.0, 1 / 2, 1 / 3, 2.0, 3.0, '1t') }, { 'layer_name': 'Conv11', 'feature_dim_hw': (10, 10), 'bbox_size': (105, 105), 'aspect_ratio': (1.0, 1 / 2, 1 / 3, 2.0, 3.0, '1t') }, { 'layer_name': 'Conv14_2', 'feature_dim_hw': (5, 5), 'bbox_size': (150, 150), 'aspect_ratio': (1.0, 1 / 2, 1 / 3, 2.0, 3.0, '1t') }, { 'layer_name': 'Conv15_2', 'feature_dim_hw': (3, 3), 'bbox_size': (195, 195), 'aspect_ratio': (1.0, 1 / 2, 1 / 3, 2.0, 3.0, '1t') }, { 'layer_name': 'Conv16_2', 'feature_dim_hw': (2, 2), 'bbox_size': (240, 240), 'aspect_ratio': (1.0, 1 / 2, 1 / 3, 2.0, 3.0, '1t') }, { 'layer_name': 'Conv17_2', 'feature_dim_hw': (1, 1), 'bbox_size': (285, 285), 'aspect_ratio': (1.0, 1 / 2, 1 / 3, 2.0, 3.0, '1t') }] prior_bboxes = generate_prior_bboxes(prior_layer_cfg) # loading the test image img_file_path = sys.argv[1] # img_file_path = 'image.png' img = Image.open(img_file_path) img = img.resize((300, 300)) plot_img = img.copy() img_array = np.asarray(img)[:, :, :3] mean = np.asarray((127, 127, 127)) std = 128.0 img_array = (img_array - mean) / std h, w, c = img_array.shape[0], img_array.shape[1], img_array.shape[2] img_tensor = torch.Tensor(img_array) test_input = img_tensor.view(1, c, h, w) # # loading test input to run test on # test_data_loader = torch.utils.data.DataLoader(test_input, # batch_size=1, # shuffle=True, # num_workers=0) # idx, (img) = next(enumerate(test_data_loader)) # # Setting model to evaluate mode net = SSD(2) test_net_state = torch.load('ssd_net.pth') net.load_state_dict(test_net_state) # net.eval() net.cuda() # Forward test_input = Variable(test_input.cuda()) test_cof, test_loc = net.forward(test_input) test_loc = test_loc.detach() test_loc_clone = test_loc.clone() # normalizing the loss to add up to 1 (for probability) test_cof_score = F.softmax(test_cof[0], dim=1) # print(test_cof_score.shape) # print(test_cof_score) # running NMS sel_idx = nms_bbox1(test_loc_clone[0], prior_bboxes, test_cof_score.detach(), overlap_threshold=0.5, prob_threshold=0.24) test_loc = loc2bbox(test_loc[0], prior_bboxes) test_loc = center2corner(test_loc) sel_bboxes = test_loc[sel_idx] # plotting the output plot_output(plot_img, sel_bboxes.cpu().detach().numpy())
def __getitem__(self, idx): """ Load the data from list, and match the ground-truth bounding boxes with prior bounding boxes. :return bbox_tensor: matched bounding box, dim: (num_priors, 4) :return bbox_label: matched classification label, dim: (num_priors) """ # TODO: implement data loading # 1. Load image as well as the bounding box with its label # 2. Normalize the image with self.mean and self.std # 3. Convert the bounding box from corner form (left-top, right-bottom): [(x,y), (x+w, y+h)] to # center form: [(center_x, center_y, w, h)] # 4. Normalize the bounding box position value from 0 to 1 item = self.dataset_list[idx] image_path = item['image_path'] labels = np.asarray(item['labels']) labels = torch.Tensor(labels).cuda() locations = torch.Tensor(item['bboxes']).cuda() bbox = np.array(item['bboxes']) image = Image.open(image_path) self.imgWidth, self.imgHeight = image.size self.resize_ratio = min(self.imgHeight / 300., self.imgWidth / 300.) locations = helper.corner2center(locations) image = self.resize(image) locations = self.resize(locations) # Prepare image array first to update crop. image = self.crop(image) image = self.brighten(image) image = self.normalize(image) # Prepare labels second to apply crop. locations = self.crop(locations) locations = self.normalize(locations) # convert to tensor img_tensor = image.view( (image.shape[2], image.shape[0], image.shape[1])) img_tensor = img_tensor.cuda() labels = labels[self.ios_index] # 4. Do the augmentation if needed. e.g. random clip the bounding box or flip the bounding box # 5. Do the matching prior and generate ground-truth labels as well as the boxes bbox_tensor, bbox_label_tensor = match_priors( self.prior_bboxes, helper.center2corner(locations), labels, iou_threshold=0.5) # [DEBUG] check the output. # assert isinstance(bbox_label_tensor, torch.Tensor) # assert isinstance(bbox_tensor, torch.Tensor) # assert bbox_tensor.dim() == 2 # assert bbox_tensor.shape[1] == 4 # assert bbox_label_tensor.dim() == 1 # assert bbox_label_tensor.shape[0] == bbox_tensor.shape[0] return img_tensor, bbox_tensor, bbox_label_tensor