def generate_all_anchors(self, im_c, size):
if self.image_center == im_c and self.size == size:
return False
self.image_center = im_c
self.size = size
a0x = im_c - size // 2 * self.stride
ori = np.array([a0x] * 4, dtype=np.float32)
zero_anchors = self.anchors + ori
x1 = zero_anchors[:, 0]
y1 = zero_anchors[:, 1]
x2 = zero_anchors[:, 2]
y2 = zero_anchors[:, 3]
x1, y1, x2, y2 = map(lambda x: x.reshape(self.anchor_num, 1, 1),
[x1, y1, x2, y2])
cx, cy, w, h = corner2center([x1, y1, x2, y2])
disp_x = np.arange(0, size).reshape(1, 1, -1) * self.stride
disp_y = np.arange(0, size).reshape(1, -1, 1) * self.stride
cx = cx + disp_x
cy = cy + disp_y
# broadcast
zero = np.zeros((self.anchor_num, size, size), dtype=np.float32)
cx, cy, w, h = map(lambda x: x + zero, [cx, cy, w, h])
x1, y1, x2, y2 = center2corner([cx, cy, w, h])
self.all_anchors = np.stack([x1, y1, x2, y2]), np.stack([cx, cy, w, h])
return True
def crop(self, inp):
# Case for image input.
if inp.shape == torch.Size([self.imgHeight, self.imgWidth, 3]):
image = inp
# Return 300x300 patch if no object is detected.
if self.locations is None:
return image[0:300, 0:300, :]
# Check the ios of the cropped image with oracle bounding box to ensure at least one labeled item.
found = False
cnt = 0
while not found:
cnt += 1
if cnt > 300:
self.crop_coordinates = torch.Tensor([150, 0, 450, 300])
image = image[int(self.crop_coordinates[1]
):int(self.crop_coordinates[3]),
int(self.crop_coordinates[0]
):int(self.crop_coordinates[2]), :]
break
crop = random.randint(0, self.imgWidth - 300)
self.crop_coordinates = torch.Tensor(
[crop, 0, crop + 300, 300])
for location in self.locations:
if helper.ios(location,
helper.corner2center(self.crop_coordinates)
) > self.cropping_ios_threshold:
found = True
image = image[int(self.crop_coordinates[1]
):int(self.crop_coordinates[3]),
int(self.crop_coordinates[0]
):int(self.crop_coordinates[2]), :]
break
return image
# Case for location input.
locations = inp
locations[:, 0] -= self.crop_coordinates[0]
# Set locations to 0 if the ios is too small.
ios = helper.ios(locations, torch.Tensor([150, 150, 300, 300]))
self.ios_index = ios > self.cropping_ios_threshold
locations[ios <= self.cropping_ios_threshold] = 0
locations = locations[self.ios_index]
# Clip the location.
locations = helper.center2corner(locations)
locations = torch.clamp(locations, 0, 300)
locations = helper.corner2center(locations)
# Save the oracle locations.
self.locations = locations
return locations
def test_corner2(self):
prior_layer_cfg = [{
'layer_name': 'Conv5',
'feature_dim_hw': (19, 19),
'bbox_size': (60, 60),
'aspect_ratio': (1.0, 1 / 2, 1 / 3, 2.0, 3.0, '1t')
}, {
'layer_name': 'Conv11',
'feature_dim_hw': (10, 10),
'bbox_size': (105, 105),
'aspect_ratio': (1.0, 1 / 2, 1 / 3, 2.0, 3.0, '1t')
}, {
'layer_name': 'Conv14_2',
'feature_dim_hw': (5, 5),
'bbox_size': (150, 150),
'aspect_ratio': (1.0, 1 / 2, 1 / 3, 2.0, 3.0, '1t')
}, {
'layer_name': 'Conv15_2',
'feature_dim_hw': (3, 3),
'bbox_size': (195, 195),
'aspect_ratio': (1.0, 1 / 2, 1 / 3, 2.0, 3.0, '1t')
}, {
'layer_name': 'Conv16_2',
'feature_dim_hw': (2, 2),
'bbox_size': (240, 240),
'aspect_ratio': (1.0, 1 / 2, 1 / 3, 2.0, 3.0, '1t')
}, {
'layer_name': 'Conv17_2',
'feature_dim_hw': (1, 1),
'bbox_size': (285, 285),
'aspect_ratio': (1.0, 1 / 2, 1 / 3, 2.0, 3.0, '1t')
}]
pp = generate_prior_bboxes(prior_layer_cfg)
print('original', pp[0])
test = center2corner(pp[0])
print('corner', test)
test = corner2center(test)
print('center', test)
torch.set_default_tensor_type('torch.cuda.FloatTensor')
print('Pytorch CUDA Enabled?:', torch.cuda.is_available())
b = 0.5 * torch.eye(3)
b_gpu = b.cuda()
print(b_gpu)
test_img = np.array(test_img)
test_img = np.subtract(test_img, [127, 127, 127])
test_img = np.divide(test_img, 128)
test_img = test_img.reshape(
(test_img.shape[2], test_img.shape[0], test_img.shape[1]))
test_img = torch.Tensor(test_img)
test_img = test_img.unsqueeze(0).cpu()
images = Variable(test_img) # Use Variable(*) to allow gradient flow\n",
conf_preds, loc_preds = net.forward(images) # Forward once\n",
print(conf_preds.shape)
print(loc_preds.shape)
bbox = bbox_helper.loc2bbox(loc_preds, prior_bboxes)
bbox = bbox[0].detach()
bbox_corner = bbox_helper.center2corner(bbox)
print(bbox_corner)
print(conf_preds)
print(bbox_corner.shape)
bbox_corner = bbox_corner
conf_preds = conf_preds[0].detach()
# idx = conf_preds[:, 2] > 0.6
# bbox_corner = bbox_corner[idx]
# bbox = bbox[idx]
# print(bbox_corner)
bbox_nms = bbox_helper.nms_bbox(bbox_corner, conf_preds)
print(bbox_nms[0])
bbox_nms = torch.Tensor(bbox_nms[0])
print(bbox_nms.shape)
def preview(self, index=0):
if index == -1:
index = random.randint(0, len(self.dataset_list))
# Acquire preview targets.
image, confidences, locations = self[index]
image = image.view(image.shape[1], image.shape[2], image.shape[0])
# Denormalize the data.
image = self.denormalize(image)
locations = self.denormalize(locations)
# Prepare data for plotting.
image = np.array(image).astype(int)
fig, ax = plt.subplots(1)
ax.imshow(image)
matched_rect, oracle_rect = None, None
# Display matched bounding boxes.
for i_location in range(0, locations.shape[0]):
if confidences[i_location] != 0:
corner = helper.center2corner(locations[i_location])
x = corner[0]
y = corner[1]
matched_rect = patches.Rectangle((x, y),
locations[i_location][2],
locations[i_location][3],
linewidth=1,
edgecolor='r',
facecolor='none')
ax.add_patch(matched_rect)
# Display ground truth bounding boxes.
for i_location in range(0, self.locations.shape[0]):
corner = helper.center2corner(self.locations[i_location])
x = corner[0]
y = corner[1]
oracle_rect = patches.Rectangle((x, y),
self.locations[i_location][2],
self.locations[i_location][3],
linewidth=1,
edgecolor='g',
facecolor='none')
ax.add_patch(oracle_rect)
fig.canvas.set_window_title('Preview at Index [' + str(index) + ']')
plt.title('Preview at Index [' + str(index) + ']')
plt.xlim(0, 300)
plt.ylim(300, 0)
if matched_rect is not None:
plt.legend([oracle_rect, matched_rect],
['Oracle Box', 'Matched Box'],
bbox_to_anchor=(1.05, 1),
loc=2,
borderaxespad=0.)
else:
plt.legend([oracle_rect], ['Oracle Box'],
bbox_to_anchor=(1.05, 1),
loc=2,
borderaxespad=0.)
plt.tight_layout()
# Output stats.
print('Available classes:', self.classes)
print('Number of oracle bounding boxes:', self.locations.shape[0])
print('Number of matched bounding boxes:', locations.shape[0])
print('Bounding box configuration:')
print(np.array(self.prior_layer_cfg))
plt.show()
def test_random2(self):
torch.set_default_tensor_type('torch.cuda.FloatTensor')
torch.set_printoptions(precision=10)
prior_layer_cfg = [{
'layer_name': 'Conv5',
'feature_dim_hw': (19, 19),
'bbox_size': (60, 60),
'aspect_ratio': (1.0, 1 / 2, 1 / 3, 2.0, 3.0, '1t')
}, {
'layer_name': 'Conv11',
'feature_dim_hw': (10, 10),
'bbox_size': (105, 105),
'aspect_ratio': (1.0, 1 / 2, 1 / 3, 2.0, 3.0, '1t')
}, {
'layer_name': 'Conv14_2',
'feature_dim_hw': (5, 5),
'bbox_size': (150, 150),
'aspect_ratio': (1.0, 1 / 2, 1 / 3, 2.0, 3.0, '1t')
}, {
'layer_name': 'Conv15_2',
'feature_dim_hw': (3, 3),
'bbox_size': (195, 195),
'aspect_ratio': (1.0, 1 / 2, 1 / 3, 2.0, 3.0, '1t')
}, {
'layer_name': 'Conv16_2',
'feature_dim_hw': (2, 2),
'bbox_size': (240, 240),
'aspect_ratio': (1.0, 1 / 2, 1 / 3, 2.0, 3.0, '1t')
}, {
'layer_name': 'Conv17_2',
'feature_dim_hw': (1, 1),
'bbox_size': (285, 285),
'aspect_ratio': (1.0, 1 / 2, 1 / 3, 2.0, 3.0, '1t')
}]
pp = generate_prior_bboxes(prior_layer_cfg)
# test_list = load_data('../Debugimage', '../Debuglabel')
test_list = load_data('../cityscapes_samples',
'../cityscapes_samples_labels')
#print(test_list)
test_dataset = CityScapeDataset(test_list)
test_data_loader = torch.utils.data.DataLoader(test_dataset,
batch_size=1,
shuffle=True,
num_workers=0)
lfw_dataset_dir = '../'
test_net = ssd_net.SSD(3)
test_net_state = torch.load(
os.path.join(lfw_dataset_dir, 'ssd_net.pth'))
test_net.load_state_dict(test_net_state)
idx, (img, bbox, label) = next(enumerate(test_data_loader))
pred_cof, pred_loc = test_net.forward(img)
print(pred_loc.shape)
import torch.nn.functional as F
pred_loc = pred_loc.detach()
bbox_center = loc2bbox(pred_loc[0], pp)
pred_cof = F.softmax(pred_cof[0])
ind = np.where(pred_cof > 0.7)
# pred_cof = F.softmax(pred_cof[ind[0]])
bbox_center = bbox_center[ind[0]]
print(ind, pred_cof)
img = img[0].cpu().numpy()
img = img.reshape((300, 300, 3))
img = (img * 128 + np.asarray([[127, 127, 127]])) / 255
fig, ax = plt.subplots(1)
imageB_array = resize(img, (600, 1200), anti_aliasing=True)
ax.imshow(imageB_array, cmap='brg')
bbox_corner = center2corner(bbox_center)
for i in range(0, bbox_corner.shape[0]):
# print('i point', bbox_corner[i, 0]*600, bbox_corner[i, 1]*300,(bbox_corner[i, 2]-bbox_corner[i, 0])*600, (bbox_corner[i, 3]-bbox_corner[i, 1])*300)
rect = patches.Rectangle(
(bbox_corner[i, 0] * 1200, bbox_corner[i, 1] * 600),
(bbox_corner[i, 2] - bbox_corner[i, 0]) * 1200,
(bbox_corner[i, 3] - bbox_corner[i, 1]) * 600,
linewidth=2,
edgecolor='r',
facecolor='none') # Create a Rectangle patch
ax.add_patch(rect) # Add the patch to the Axes
plt.show()
def test_dataLoad(self):
torch.set_default_tensor_type('torch.cuda.FloatTensor')
torch.set_printoptions(precision=10)
prior_layer_cfg = [{
'layer_name': 'Conv5',
'feature_dim_hw': (19, 19),
'bbox_size': (60, 60),
'aspect_ratio': (1.0, 1 / 2, 1 / 3, 2.0, 3.0, '1t')
}, {
'layer_name': 'Conv11',
'feature_dim_hw': (10, 10),
'bbox_size': (105, 105),
'aspect_ratio': (1.0, 1 / 2, 1 / 3, 2.0, 3.0, '1t')
}, {
'layer_name': 'Conv14_2',
'feature_dim_hw': (5, 5),
'bbox_size': (150, 150),
'aspect_ratio': (1.0, 1 / 2, 1 / 3, 2.0, 3.0, '1t')
}, {
'layer_name': 'Conv15_2',
'feature_dim_hw': (3, 3),
'bbox_size': (195, 195),
'aspect_ratio': (1.0, 1 / 2, 1 / 3, 2.0, 3.0, '1t')
}, {
'layer_name': 'Conv16_2',
'feature_dim_hw': (2, 2),
'bbox_size': (240, 240),
'aspect_ratio': (1.0, 1 / 2, 1 / 3, 2.0, 3.0, '1t')
}, {
'layer_name': 'Conv17_2',
'feature_dim_hw': (1, 1),
'bbox_size': (285, 285),
'aspect_ratio': (1.0, 1 / 2, 1 / 3, 2.0, 3.0, '1t')
}]
pp = generate_prior_bboxes(prior_layer_cfg)
#test_list = load_data('../Debugimage', '../Debuglabel')
test_list = load_data('../cityscapes_samples',
'../cityscapes_samples_labels')
print(test_list)
gt_bbox = np.asarray(test_list[0]['label'][1]) * [
300 / 2048, 300 / 1024, 300 / 2048, 300 / 1024
]
print('ground truth from file:', test_list[0]['label'][0])
test_dataset = CityScapeDataset(test_list)
test_data_loader = torch.utils.data.DataLoader(test_dataset,
batch_size=1,
shuffle=True,
num_workers=0)
idx, (img, bbox, label) = next(enumerate(test_data_loader))
bbox = bbox[0]
label = label[0]
print(bbox.shape, label.shape)
print('matched label', label[np.where(label > 0)], np.where(label > 0),
label.shape)
print('first bbox from data_set:', bbox[0], label[0])
bbox_center = loc2bbox(bbox, pp)
bbox_corner = center2corner(bbox_center)
img = img[0].cpu().numpy()
img = img.reshape((300, 300, 3))
img = (img * 128 + np.asarray([[127, 127, 127]])) / 255
# for i in range(0, bbox.shape[0]):
# cv2.rectangle(img, (bbox[i,0], bbox[i,1]), (bbox[i,2], bbox[i,3]), (0, 255, 0), 3)
#cv2.imshow("img", img)
# Create figure and axes
fig, ax = plt.subplots(1)
imageB_array = resize(img, (300, 300), anti_aliasing=True)
ax.imshow(imageB_array, cmap='brg')
bbox_corner = bbox_corner.cpu().numpy()
bbox_corner = bbox_corner[np.where(label > 0)]
temp_lab = label[np.where(label > 0)]
print('matched bbox ======', bbox_corner)
pp = center2corner(pp)
pp = pp[np.where(label > 0)]
print('864 tensor: ', pp)
for i in range(0, bbox_corner.shape[0]):
if temp_lab[i] == 1:
# print('i point', bbox_corner[i, 0]*600, bbox_corner[i, 1]*300,(bbox_corner[i, 2]-bbox_corner[i, 0])*600, (bbox_corner[i, 3]-bbox_corner[i, 1])*300)
rect = patches.Rectangle(
(bbox_corner[i, 0] * 300, bbox_corner[i, 1] * 300),
(bbox_corner[i, 2] - bbox_corner[i, 0]) * 300,
(bbox_corner[i, 3] - bbox_corner[i, 1]) * 300,
linewidth=2,
edgecolor='r',
facecolor='none') # Create a Rectangle patch
ax.add_patch(rect) # Add the patch to the Axes
else:
rect = patches.Rectangle(
(bbox_corner[i, 0] * 1200, bbox_corner[i, 1] * 600),
(bbox_corner[i, 2] - bbox_corner[i, 0]) * 1200,
(bbox_corner[i, 3] - bbox_corner[i, 1]) * 600,
linewidth=2,
edgecolor='y',
facecolor='none') # Create a Rectangle patch
ax.add_patch(rect) # Add the patch to the Axes
for i in range(0, pp.shape[0]):
rect = patches.Rectangle(
(pp[i, 0] * 300, pp[i, 1] * 300), (pp[i, 2] - pp[i, 0]) * 300,
(pp[i, 3] - pp[i, 1]) * 300,
linewidth=1,
edgecolor='blue',
facecolor='none') # Create a Rectangle patch
ax.add_patch(rect) # Add the patch to the Axes
# for i in range(0, gt_bbox.shape[0]):
# rect = patches.Rectangle((gt_bbox[i][0], gt_bbox[i][1]),
# (gt_bbox[i][2] - gt_bbox[i][0]),
# (gt_bbox[i][3] - gt_bbox[i][1]), linewidth=1, edgecolor='g',
# facecolor='none') # Create a Rectangle patch
# ax.add_patch(rect) # Add the patch to the Axes
plt.show()
else:
test_net_state = torch.load(path_to_trained_model, map_location='cpu')
test_net = SSD(num_classes=3)
test_net.load_state_dict(test_net_state)
test_net.eval()
test_image_permuted = img_tensor.permute(0, 3, 1, 2)
test_image_permuted = Variable(test_image_permuted.float())
test_conf_preds, test_loc_preds = test_net.forward(test_image_permuted)
test_bbox_priors = prior_bboxes.unsqueeze(0)
test_bbox_preds = loc2bbox(test_loc_preds.cpu(), test_bbox_priors.cpu(), center_var=0.1, size_var=0.2)
sel_bbox_preds = nms_bbox(test_bbox_preds.squeeze().detach(), test_conf_preds.squeeze().detach().cpu(), overlap_threshold=0.5, prob_threshold=0.5)
rects = []
classes = []
for key in sel_bbox_preds.keys():
for value in sel_bbox_preds[key]:
classes.append(key)
rects.append(value)
rects = center2corner(torch.tensor(rects))*300
if len(rects) != 0:
if rects.dim() == 1:
rects = rects.unsqueeze(0)
classes = torch.tensor(classes)
drawRectsWithImgPLT(img, rects, classes)
else:
print("no object detected in this img")
def main():
torch.set_default_tensor_type('torch.cuda.FloatTensor')
prior_layer_cfg = [{
'layer_name': 'Conv5',
'feature_dim_hw': (19, 19),
'bbox_size': (60, 60),
'aspect_ratio': (1.0, 1 / 2, 1 / 3, 2.0, 3.0, '1t')
}, {
'layer_name': 'Conv11',
'feature_dim_hw': (10, 10),
'bbox_size': (105, 105),
'aspect_ratio': (1.0, 1 / 2, 1 / 3, 2.0, 3.0, '1t')
}, {
'layer_name': 'Conv14_2',
'feature_dim_hw': (5, 5),
'bbox_size': (150, 150),
'aspect_ratio': (1.0, 1 / 2, 1 / 3, 2.0, 3.0, '1t')
}, {
'layer_name': 'Conv15_2',
'feature_dim_hw': (3, 3),
'bbox_size': (195, 195),
'aspect_ratio': (1.0, 1 / 2, 1 / 3, 2.0, 3.0, '1t')
}, {
'layer_name': 'Conv16_2',
'feature_dim_hw': (2, 2),
'bbox_size': (240, 240),
'aspect_ratio': (1.0, 1 / 2, 1 / 3, 2.0, 3.0, '1t')
}, {
'layer_name': 'Conv17_2',
'feature_dim_hw': (1, 1),
'bbox_size': (285, 285),
'aspect_ratio': (1.0, 1 / 2, 1 / 3, 2.0, 3.0, '1t')
}]
prior_bboxes = generate_prior_bboxes(prior_layer_cfg)
# loading the test image
img_file_path = sys.argv[1]
# img_file_path = 'image.png'
img = Image.open(img_file_path)
img = img.resize((300, 300))
plot_img = img.copy()
img_array = np.asarray(img)[:, :, :3]
mean = np.asarray((127, 127, 127))
std = 128.0
img_array = (img_array - mean) / std
h, w, c = img_array.shape[0], img_array.shape[1], img_array.shape[2]
img_tensor = torch.Tensor(img_array)
test_input = img_tensor.view(1, c, h, w)
# # loading test input to run test on
# test_data_loader = torch.utils.data.DataLoader(test_input,
# batch_size=1,
# shuffle=True,
# num_workers=0)
# idx, (img) = next(enumerate(test_data_loader))
# # Setting model to evaluate mode
net = SSD(2)
test_net_state = torch.load('ssd_net.pth')
net.load_state_dict(test_net_state)
# net.eval()
net.cuda()
# Forward
test_input = Variable(test_input.cuda())
test_cof, test_loc = net.forward(test_input)
test_loc = test_loc.detach()
test_loc_clone = test_loc.clone()
# normalizing the loss to add up to 1 (for probability)
test_cof_score = F.softmax(test_cof[0], dim=1)
# print(test_cof_score.shape)
# print(test_cof_score)
# running NMS
sel_idx = nms_bbox1(test_loc_clone[0],
prior_bboxes,
test_cof_score.detach(),
overlap_threshold=0.5,
prob_threshold=0.24)
test_loc = loc2bbox(test_loc[0], prior_bboxes)
test_loc = center2corner(test_loc)
sel_bboxes = test_loc[sel_idx]
# plotting the output
plot_output(plot_img, sel_bboxes.cpu().detach().numpy())
def __getitem__(self, idx):
"""
Load the data from list, and match the ground-truth bounding boxes with prior bounding boxes.
:return bbox_tensor: matched bounding box, dim: (num_priors, 4)
:return bbox_label: matched classification label, dim: (num_priors)
"""
# TODO: implement data loading
# 1. Load image as well as the bounding box with its label
# 2. Normalize the image with self.mean and self.std
# 3. Convert the bounding box from corner form (left-top, right-bottom): [(x,y), (x+w, y+h)] to
# center form: [(center_x, center_y, w, h)]
# 4. Normalize the bounding box position value from 0 to 1
item = self.dataset_list[idx]
image_path = item['image_path']
labels = np.asarray(item['labels'])
labels = torch.Tensor(labels).cuda()
locations = torch.Tensor(item['bboxes']).cuda()
bbox = np.array(item['bboxes'])
image = Image.open(image_path)
self.imgWidth, self.imgHeight = image.size
self.resize_ratio = min(self.imgHeight / 300., self.imgWidth / 300.)
locations = helper.corner2center(locations)
image = self.resize(image)
locations = self.resize(locations)
# Prepare image array first to update crop.
image = self.crop(image)
image = self.brighten(image)
image = self.normalize(image)
# Prepare labels second to apply crop.
locations = self.crop(locations)
locations = self.normalize(locations)
# convert to tensor
img_tensor = image.view(
(image.shape[2], image.shape[0], image.shape[1]))
img_tensor = img_tensor.cuda()
labels = labels[self.ios_index]
# 4. Do the augmentation if needed. e.g. random clip the bounding box or flip the bounding box
# 5. Do the matching prior and generate ground-truth labels as well as the boxes
bbox_tensor, bbox_label_tensor = match_priors(
self.prior_bboxes,
helper.center2corner(locations),
labels,
iou_threshold=0.5)
# [DEBUG] check the output.
# assert isinstance(bbox_label_tensor, torch.Tensor)
# assert isinstance(bbox_tensor, torch.Tensor)
# assert bbox_tensor.dim() == 2
# assert bbox_tensor.shape[1] == 4
# assert bbox_label_tensor.dim() == 1
# assert bbox_label_tensor.shape[0] == bbox_tensor.shape[0]
return img_tensor, bbox_tensor, bbox_label_tensor
