def _data_aug_fn(image, ground_truth):
"""Data augmentation function."""
ground_truth = cPickle.loads(ground_truth)
ground_truth = list(ground_truth)
annos = ground_truth[0]
mask = ground_truth[1]
h_mask, w_mask, _ = np.shape(image)
# mask
mask_miss = np.ones((h_mask, w_mask), dtype=np.uint8)
for seg in mask:
bin_mask = maskUtils.decode(seg)
bin_mask = np.logical_not(bin_mask)
mask_miss = np.bitwise_and(mask_miss, bin_mask)
## image data augmentation
# randomly resize height and width independently, scale is changed
image, annos, mask_miss = keypoint_random_resize(image,
annos,
mask_miss,
zoom_range=(0.8, 1.2))
# random rotate
image, annos, mask_miss = keypoint_random_rotate(image,
annos,
mask_miss,
rg=15.0)
# random left-right flipping
image, annos, mask_miss = keypoint_random_flip(image,
annos,
mask_miss,
prob=0.5)
# random resize height and width together
image, annos, mask_miss = keypoint_random_resize_shortestedge(
image, annos, mask_miss, min_size=(hin, win), zoom_range=(0.95, 1.6))
# random crop
image, annos, mask_miss = keypoint_random_crop(image,
annos,
mask_miss,
size=(hin,
win)) # with padding
# generate result maps including keypoints heatmap, pafs and mask
h, w, _ = np.shape(image)
height, width, _ = np.shape(image)
heatmap = get_heatmap(annos, height, width)
vectormap = get_vectormap(annos, height, width)
resultmap = np.concatenate((heatmap, vectormap), axis=2)
image = np.array(image, dtype=np.float32)
img_mask = mask_miss.reshape(hin, win, 1)
image = image * np.repeat(img_mask, 3, 2)
resultmap = np.array(resultmap, dtype=np.float32)
mask_miss = cv2.resize(mask_miss, (hout, wout),
interpolation=cv2.INTER_AREA)
mask_miss = np.array(mask_miss, dtype=np.float32)
return image, resultmap, mask_miss
def slicing(features, seeds_features, seeds_label, label_map, adjacency,
sigma=1., resize_shape=(480, 854)):
"""
Propagate the labels of seeds to pixels.
Similiar with the slicing step in bilateral filtering.
Args:
features: dense feature map [height, width, f_dim]
seeds_features: [num_seeds_cur_prev_following_frame, f_dim]
seeds_label: [num_seeds_cur_prev_following_frame]
label_map: [height, width]
adjacency: [num_seeds_current, num_seeds_cur_prev_following_frame]
sigma: Used to compute the soft prob of FG
resize_shape: Resize to original resolution
Returns:
prob: soft FG/BG probability map
dist_vis: visualized distance map
"""
label_map_flatten = np.reshape(label_map, [-1])
num_seeds = np.max(label_map)+1
# Label_map_one_hot [num_pixels, num_seeds_current]
label_map_one_hot = np.zeros((label_map_flatten.shape[0], num_seeds), dtype=np.int16)
label_map_one_hot[np.arange(label_map_flatten.shape[0]), label_map_flatten] = 1
# weight_idx: [num_pixels, num_seeds_cur_prev_following_frame]
# Only neighbouring seeds have weights > 0
weight_idx = np.matmul(label_map_one_hot, adjacency)
feature_dim = features.shape[2]
# This implementation is not very efficient
# It computes pairwise distance between all pixels and all seeds (from 3 frames)
# dist: [num_pixels, num_seeds_cur_prev_following_frame]
dist = euclidean_distances(np.reshape(features, [-1, feature_dim]), seeds_features)
weight = np.exp(-dist*dist/sigma/sigma)
weight *= weight_idx
fg_votes = np.max(weight*np.expand_dims(seeds_label==1, 0), axis=1)
bg_votes = np.max(weight*np.expand_dims(seeds_label==0, 0), axis=1)
height = features.shape[0]
width = features.shape[1]
fg_votes = fg_votes.reshape((height, width))+1e-8
bg_votes = bg_votes.reshape((height, width))+1e-8
fg_votes = cv2.resize(fg_votes, (resize_shape[1], resize_shape[0]),
interpolation=cv2.INTER_LINEAR)
bg_votes = cv2.resize(bg_votes, (resize_shape[1], resize_shape[0]),
interpolation=cv2.INTER_LINEAR)
prob = np.stack([bg_votes, fg_votes], axis=2)
dist_vis = utils.get_heatmap(np.concatenate([fg_votes, bg_votes], axis=0))
prob = prob/np.sum(prob, axis=2, keepdims=True)
return prob, dist_vis
def detect(self, image, do_hog_once=True):
out_windows = []
if do_hog_once:
out_windows = self.find_multiscale(image)
heatmap = utils.get_heatmap(image, out_windows)
heatmap = self.smoother(heatmap)
heatmap = utils.apply_threshold(heatmap,3)
boxes = utils.get_labeled_boxes(heatmap)
else:
windows = utils.slide_window(image, x_start_stop=[None, None], y_start_stop=self.y_start_stop,
xy_window=(64, 64), xy_overlap=(0.85, 0.85))
boxes = self.search_windows(image, windows)
final_image = utils.draw_boxes(image, boxes, color=(0, 0, 255), thick=6)
return final_image
def _data_aug_fn(image, annos, mask_miss):
## image data augmentation
# randomly resize height and width independently, scale is changed
image, annos, mask_miss = keypoint_random_resize(image,
annos,
mask_miss,
zoom_range=(0.8, 1.2))
# random rotate
image, annos, mask_miss = keypoint_random_rotate(image,
annos,
mask_miss,
rg=15.0)
# random left-right flipping
image, annos, mask_miss = keypoint_random_flip(image,
annos,
mask_miss,
prob=0.5)
# random resize height and width together
image, annos, mask_miss = keypoint_random_resize_shortestedge(
image,
annos,
mask_miss,
min_size=(hin, win),
zoom_range=(0.95, 1.6))
# random crop
image, annos, mask_miss = keypoint_random_crop(
image, annos, mask_miss, size=(hin, win)) # with padding
# generate result maps including keypoints heatmap, pafs and mask
h, w, _ = np.shape(image)
height, width, _ = np.shape(image)
heatmap = get_heatmap(annos, height, width)
vectormap = get_vectormap(annos, height, width)
resultmap = np.concatenate((heatmap, vectormap), axis=2)
image = np.array(image, dtype=np.float32)
img_mask = mask_miss.reshape(hin, win, 1)
image = image * np.repeat(img_mask, 3, 2)
resultmap = np.array(resultmap, dtype=np.float32)
mask_miss = cv2.resize(mask_miss, (hout, wout),
interpolation=cv2.INTER_AREA)
mask_miss = np.array(mask_miss, dtype=np.float32)
return image, resultmap, mask_miss
#model_fname = 'model_fullset'
model_fname = 'model_fullset_cv2'
scaler, clf = joblib.load(model_fname + '_scaler.pickle'), \
joblib.load(model_fname + '_svc.pickle')
file_list = sorted(glob.glob('test_images/*.jpg'))
output_dir = 'output_images/'
for f in file_list:
time_start = time.time()
fname = f[f.rindex('/') + 1:]
print('Processing', fname)
img = cv2.imread(f)
boxes = utils.detect_vehicles_parallel(img, scaler, clf)
cv2.imwrite(output_dir + 'raw_boxes_' + fname, \
utils.draw_boxes(img, boxes))
heatmap = utils.get_heatmap(img.shape, boxes, 1)
cv2.imwrite(output_dir + 'hm_no_thresh_' + fname, \
(heatmap * 255).astype(np.uint8))
heatmap = utils.get_heatmap(img.shape, boxes, 3)
cv2.imwrite(output_dir + 'hm_thresh_' + fname, \
(255*heatmap).astype(np.uint8))
hmlbl, lbls = utils.get_labels(heatmap)
if lbls:
cv2.imwrite(output_dir + 'labeled_hm_' + fname, \
(hmlbl*(255//lbls)).astype(np.uint8))
else:
cv2.imwrite(output_dir + 'labeled_hm_' + fname, \
hmlbl.astype(np.uint8))
bboxes = utils.get_bboxes(hmlbl, lbls)
img_final = utils.draw_boxes(img, bboxes)
cv2.imwrite(output_dir + 'final_' + fname, img_final)
row = len(os.listdir(images_path))
col = 3
index = 0
for filename in os.listdir(images_path):
image = mpimg.imread(images_path + filename)
index += 1
plt.subplot(row, col, index)
out_windows = detector.find_multiscale(image)
window_img = utils.draw_boxes(image, out_windows, color=(0, 0, 255), thick=6)
plt.imshow(window_img)
plt.title('detected windows')
index += 1
plt.subplot(row, col, index)
heatmap = utils.get_heatmap(image, out_windows)
# Apply threshold to help remove false positives
heatmap = apply_threshold(heatmap,3)
plt.imshow(heatmap, cmap='hot')
plt.title('Heat Map')
index += 1
plt.subplot(row, col, index)
boxes = utils.get_labeled_boxes(heatmap)
final_img = utils.draw_boxes(image, boxes, color=(0, 0, 255),thick=6)
plt.imshow(final_img)
plt.title('Car position')
plt.show()
# image = mpimg.imread(images_path + 'test1.jpg')
# # search_windows = detector.draw_debug_windows(image)
def main():
parser = argparse.ArgumentParser(
description='Seg-unravel on a given image.')
parser.add_argument('--network',
type=str,
help='The segmentation network to be used.',
required=True)
parser.add_argument(
'--shift_type',
type=str,
help='The shift type to be used. Read Thesis for more details.',
default='full_shift')
parser.add_argument('--image',
type=str,
help='Path to image file.',
required=True)
args = parser.parse_args()
# set caffe before importing seg_fix
caffe_version = args.network
set_caffe_path(caffe_version)
import seg_fix
import caffe
# get the caffe model
prototxt = os.path.join('prototxt', file_map[caffe_version][0])
wts = os.path.join('weights', file_map[caffe_version][1])
if caffe_version in ['DEEPLAB_V2', 'FCN']:
net = caffe.Net(prototxt, wts, caffe.TEST)
else:
net = caffe.Net(prototxt, wts)
# For forward
image_blob = u.get_blob(args.image)
net.blobs['data'].data[...] = image_blob
last_layer = file_map[caffe_version][2]
top_layer_output = net.forward()[last_layer]
# Seg_fix in action
fixer = seg_fix.seg_fix(prototxt, caffe_version)
top_fixations = fixer.get_top_fixations(top_layer_output)
# find class-id
seg_map = np.argmax(top_layer_output, 1)[0]
class_ids = list(np.unique(seg_map))
class_ids.remove(0) # not bg
seg_space = [np.sum(seg_map == id) for id in class_ids]
class_id = 15 #class_ids[np.argmax(seg_space)]
# you can get fixations for each of the detected classes by sending each detected class-id in the following function
image_fixations, all_fixations = fixer.get_fixations_at_all_layers(
top_fixations[class_id], net, save_all=True)
# save numpy with fixations
f = open('temp.pkl', 'w')
pickle.dump(all_fixations, f)
f.close()
# Get the image_with Fixations
img_with_fixations = u.embed_fixations(args.image, image_fixations)
cv2.imwrite('temp.png', img_with_fixations)
# get the gif of fixation flow:
img_with_fixations = u.embed_fixations_gif(args.image, all_fixations,
fixer, net)
# this saves the gif as temp.gif
# heat map
u.get_heatmap(args.image, image_fixations)