def run(self):
i = 0
while True:
p = self.get_next_image_file()
if p is None:
break
img = cv2.imread(p)
if i >= self.skip:
rect = self.label_file.readline()
try:
box = map(lambda x: int(x), rect.split("\t"))
B = BoundingRegion(image_shape=img.shape, box=np.array(box))
except:
try:
box = map(lambda x: int(x), rect.split(","))
B = BoundingRegion(image_shape=img.shape, box=np.array(box))
except:
print "No more bounding boxes!"
B = BoundingRegion()
else:
B = BoundingRegion()
F = LabeledMovieFrame(internal_storage_method='jpg', compression_level=90)
F.set_image(img)
F.set_label(B)
B.draw_box(img)
cv2.imshow("image", img)
cv2.moveWindow("image", 50, 50)
cv2.waitKey(1)
i += 1
self.frame_collection.append(F)
self.frame_collection.write_to_file(self.output_file)
class StruckTracker(GenericVisionTracker):
"""
TODO
"""
def __init__(self):
"""
Initialize the tracker
"""
self.name = 'struck_tracker'
f = open("config.txt", "w")
f.write(struck_config)
f.close()
self.reset()
def reset(self):
"""
Reset the tracker
:return:
"""
self._primed = False
self.bbox = None
def _prime(self, im, bounding_region):
"""
prime tracker on image and bounding box
:param im: input image (3 - channel numpy array)
:type im: numpy.ndarray
:param bounding_region: initial bounding region of the tracked object
:type bounding_region: PVM_tools.bounding_region.BoundingRegion
"""
self.bbox=bounding_region
bounding_box = bounding_region.get_box_pixels()
struck.STRUCK_init(im, bounding_box)
def _track(self, im):
"""
Track on given image, return a bounding box
:param im: image (3 - channel numpy array)
:type im: numpy.ndarray
:return: bounding box of the tracker object
:rtype: PVM_tools.bounding_region.BoundingRegion
"""
# this is required so that tracker is re-initialized if it is primed again
self._primed = False
struck.STRUCK_track(im)
struck_bbox = struck.STRUCK_get_bbox()
self.bounding_box = [struck_bbox["xmin"], struck_bbox["ymin"], struck_bbox["width"], struck_bbox["height"]]
if self.bounding_box == ():
self.bbox = BoundingRegion()
else:
self.bbox = BoundingRegion(image_shape=(im.shape[0], im.shape[1], 3), box=self.bounding_box)
return self.bbox.copy()
def get_heatmap(self, heatmap_name=None):
return self.bbox.get_mask()
def set_bounding_box_with_center(self, x, y):
box = [
np.clip(int(x + 0.5) - self.x_size / 2, 0, self.image_shape[1]),
np.clip(int(y + 0.5) - self.y_size / 2, 0, self.image_shape[0]),
self.x_size + min(int(x + 0.5) - self.x_size / 2, 0),
self.y_size + min(int(y + 0.5) - self.y_size / 2, 0)
]
self._current_bounds = BoundingRegion(box=box)
self.needs_refresh = True
def _track(self, im):
"""
Track on given image, rseturn a bounding box
:param im: image (3 - channel numpy array)
:type im: numpy.ndarray
:return: bounding box of the tracker object
:rtype: PVM_tools.bounding_region.BoundingRegion
"""
# this is required so that tracker is re-initialized if it is primed again
current_frame = cv2.resize(im, dsize=self.image_size)
self.prop_dict['input_array'][:] = current_frame
self.prop_dict['input_array_float'][:] = current_frame.astype(np.float)/255
for i in range(self.step_per_frame):
self.executor.step()
norm = 1.0 / len(self.prop_dict['predicted_readout_arrays'])
self.readout_heatmap[:] = 0
for k in self.prop_dict['predicted_readout_arrays']:
self.readout_heatmap[:] += cv2.resize(k.view(np.ndarray), dsize=self.image_size) * norm
am = np.unravel_index(np.argmax(self.readout_heatmap), self.readout_heatmap.shape)
if len(am) == 3:
for d in range(3):
if am[2] != d:
self.readout_heatmap[:, :, d] = 0
self.heatmap = self.readout_heatmap.view(np.ndarray)
if len(self.heatmap.shape) == 3:
self.heatmap = np.max(self.heatmap, axis=2)
self.heatmap = cv2.resize(self.heatmap, dsize=(im.shape[1], im.shape[0]), interpolation=cv2.INTER_CUBIC)
self.heatmap = (self.heatmap * 255).astype(np.uint8)
if np.max(self.heatmap) > (np.median(self.heatmap)+self.threshold):
threshold=(np.max(self.heatmap)-np.median(self.heatmap))*0.5+np.median(self.heatmap)
ret, thresh = cv2.threshold(self.heatmap, threshold, 255, 0)
contours, hierarchy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
if len(contours) > 0:
max_cnt=None
for cnt in contours:
pt = np.unravel_index(np.argmax(self.heatmap), self.heatmap.shape)
if cv2.pointPolygonTest(cnt, (pt[1], pt[0]), False)>=0:
max_cnt = cnt
break
if max_cnt is None:
self.bbox = BoundingRegion()
else:
x, y, w, h = cv2.boundingRect(max_cnt)
if w*h > 25:
self.bbox = BoundingRegion(image_shape=im.shape, box=[x, y, w, h])
self.bbox.scale(1.1)
else:
self.bbox = BoundingRegion()
else:
self.bbox = BoundingRegion()
else:
self.bbox = BoundingRegion()
self._primed = False
return self.bbox.copy()
class CenterVisionTracker(GenericVisionTracker):
"""
This class exposes the null vision tracker which just always
returns its priming bounding box
"""
def __init__(self, size_factor=0.2, new_name=None):
"""
Initialize the tracker
"""
if new_name is None:
self.name = 'center_tracker'
else:
self.name = new_name
self.size_factor = size_factor
def reset(self):
"""
Reset the tracker
:return:
"""
self._primed = False
self.bbox = None
def _prime(self, im, bounding_region):
"""
prime tracker on image and bounding box
:param im: input image (3 - channel numpy array)
:type im: numpy.ndarray
:param bounding_region: initial bounding region of the tracked object
:type bounding_region: PVM_tools.bounding_region.BoundingRegion
"""
box = np.array([(im.shape[1] - im.shape[1] * self.size_factor) / 2,
(im.shape[0] - im.shape[0] * self.size_factor) / 2,
im.shape[1] * self.size_factor,
im.shape[0] * self.size_factor]).astype(np.int)
self.bbox = BoundingRegion(image_shape=im.shape, box=box)
if not self._primed:
self._primed = True
def _track(self, im):
"""
Track on given image, rseturn a bounding box
:param im: image (3 - channel numpy array)
:type im: numpy.ndarray
:return: bounding box of the tracker object
:rtype: PVM_tools.bounding_region.BoundingRegion
"""
# this is required so that tracker is re-initialized if it is primed again
self._primed = False
return self.bbox.copy()
def get_heatmap(self, heatmap_name=None):
return self.bbox.get_mask()
class CenterVisionTracker(GenericVisionTracker):
"""
This class exposes the null vision tracker which just always
returns its priming bounding box
"""
def __init__(self, size_factor=0.2, new_name=None):
"""
Initialize the tracker
"""
if new_name is None:
self.name = 'center_tracker'
else:
self.name = new_name
self.size_factor = size_factor
def reset(self):
"""
Reset the tracker
:return:
"""
self._primed = False
self.bbox = None
def _prime(self, im, bounding_region):
"""
prime tracker on image and bounding box
:param im: input image (3 - channel numpy array)
:type im: numpy.ndarray
:param bounding_region: initial bounding region of the tracked object
:type bounding_region: PVM_tools.bounding_region.BoundingRegion
"""
box = np.array([(im.shape[1]-im.shape[1]*self.size_factor)/2,
(im.shape[0]-im.shape[0]*self.size_factor)/2,
im.shape[1]*self.size_factor,
im.shape[0]*self.size_factor]).astype(np.int)
self.bbox = BoundingRegion(image_shape=im.shape, box=box)
if not self._primed:
self._primed = True
def _track(self, im):
"""
Track on given image, rseturn a bounding box
:param im: image (3 - channel numpy array)
:type im: numpy.ndarray
:return: bounding box of the tracker object
:rtype: PVM_tools.bounding_region.BoundingRegion
"""
# this is required so that tracker is re-initialized if it is primed again
self._primed = False
return self.bbox.copy()
def get_heatmap(self, heatmap_name=None):
return self.bbox.get_mask()
def process(self, heatmap, previous_bb=None, peak=None):
"""
This method computes a bounding box around the heatmap peak.
Arguments:
heatmap - target position heat map (may have multiple local maxima).
peak - [y, x] of the most likely target position (usually the highest peak of the heatmap).
This argument tells compute_bounding_box() which local maximum to choose.
returns a bounding box array (x_upper_left,y_upper_left,width,height) or None if it can't be found
"""
if np.max(heatmap) == 0.0:
return BoundingRegion()
heatmap = cv2.boxFilter(heatmap,
ddepth=-1,
ksize=(self.recovery_kernel_size,
self.recovery_kernel_size),
borderType=cv2.BORDER_REPLICATE)
if peak is None:
peak = np.unravel_index(np.argmax(heatmap), heatmap.shape)
if np.issubdtype(heatmap.dtype, np.float):
_, heatmap = cv2.threshold(heatmap,
self.heatmap_threshold * (1.0 / 255),
255, cv2.THRESH_BINARY)
else:
_, heatmap = cv2.threshold(heatmap, self.heatmap_threshold, 255,
cv2.THRESH_BINARY)
if heatmap[peak[0], peak[1]] != 255:
cors = np.nonzero(heatmap)
new_peak = None
if len(cors[0]) > 0:
dist2 = (cors[0] - peak[0])**2 + (cors[1] - peak[1])**2
ind = np.argmin(dist2)
if dist2[ind] < self.distance_threshold**2:
new_peak = np.array([cors[0][ind], cors[1][ind]])
if new_peak is None:
return BoundingRegion(image_shape=(heatmap.shape[0],
heatmap.shape[1], 3),
box=np.array([peak[1], peak[0], 1, 1]))
peak = new_peak
_, bounding_box = cv2.floodFill(heatmap,
None,
tuple(peak[::-1]),
255,
loDiff=10,
flags=cv2.FLOODFILL_FIXED_RANGE)
return BoundingRegion(image_shape=(heatmap.shape[0], heatmap.shape[1],
3),
box=np.asarray(bounding_box))
def set_bounding_box_with_center(self, x, y):
box = [np.clip(int(x+0.5)-self.x_size/2, 0, self.image_shape[1]),
np.clip(int(y+0.5)-self.y_size/2, 0, self.image_shape[0]),
self.x_size + min(int(x+0.5)-self.x_size/2, 0),
self.y_size + min(int(y+0.5)-self.y_size/2, 0)]
self._current_bounds = BoundingRegion(box=box)
self.needs_refresh = True
def create_windows(self):
''' Create playback progress bar (allow user to move along the movie) '''
cv2.namedWindow(self.win_name)
cv2.moveWindow(self.win_name, 150, 150)
cv2.namedWindow("completeness")
cv2.moveWindow("completeness", 150, 50)
cv2.setMouseCallback(self.win_name, self.trackbar_window_onMouse)
cv2.createTrackbar("Frame", self.win_name, 0,
len(self.movie) - 1, self.jump_to_frame_callback)
cv2.createTrackbar("Trim start", self.win_name, 0,
len(self.movie) - 1, self.set_trim_start)
cv2.createTrackbar("Trim end", self.win_name,
len(self.movie) - 1,
len(self.movie) - 1, self.set_trim_end)
im_height = 30 # 30 pixels high completeness bar
im_width = len(self.movie)
self.completeness_image = np.zeros((im_height, im_width, 3))
current_frame_index = int(im_width * float(self.current_frame_index) /
len(self.movie))
self.completeness_image[:, current_frame_index, :] = 1.
self.completeness = np.zeros(im_width, dtype=np.uint8)
for i in xrange(len(self.movie)):
bounds = self.movie.Frame(i).get_label(channel=self.channel,
target=self.target)
if bounds is None or bounds.empty:
if bounds is None:
self.movie.Frame(i).set_label(channel=self.channel,
target=self.target,
label=BoundingRegion())
self.completeness[i] = 2
else:
if bounds.is_keyframe():
self.completeness[i] = 1
else:
self.completeness[i] = 0
def _prime(self, im, bounding_region):
"""
prime tracker on image and bounding box
:param im: input image (3 - channel numpy array)
:type im: numpy.ndarray
:param bounding_region: initial bounding region of the tracked object
:type bounding_region: PVM_tools.bounding_region.BoundingRegion
"""
box = np.array([(im.shape[1] - im.shape[1] * self.size_factor) / 2,
(im.shape[0] - im.shape[0] * self.size_factor) / 2,
im.shape[1] * self.size_factor,
im.shape[0] * self.size_factor]).astype(np.int)
self.bbox = BoundingRegion(image_shape=im.shape, box=box)
if not self._primed:
self._primed = True
def _track(self, im):
"""
Track
:param im: image
:type im: numpy.ndarray
:return: bounding region
:rtype: PVM_tools.bounding_region.BoundingRegion
"""
if not self._primed:
return BoundingRegion()
self.prob = self._back_projector.calculate(im)
bounding_box = self._bounding_boxer.process(heatmap=self.prob)
if bounding_box.confidence > self._confidence_threshold:
return bounding_box.copy()
else:
return BoundingRegion()
def _track(self, im):
"""
Track on given image, return a bounding box
:param im: image (3 - channel numpy array)
:type im: numpy.ndarray
:return: bounding box of the tracker object
:rtype: PVM_tools.bounding_region.BoundingRegion
"""
# this is required so that tracker is re-initialized if it is primed again
self._primed = False
struck.STRUCK_track(im)
struck_bbox = struck.STRUCK_get_bbox()
self.bounding_box = [struck_bbox["xmin"], struck_bbox["ymin"], struck_bbox["width"], struck_bbox["height"]]
if self.bounding_box == ():
self.bbox = BoundingRegion()
else:
self.bbox = BoundingRegion(image_shape=(im.shape[0], im.shape[1], 3), box=self.bounding_box)
return self.bbox.copy()
def _track(self, im):
"""
Track on given image, return a bounding box
:param im: image (3 - channel numpy array)
:type im: numpy.ndarray
:return: bounding box of the tracker object
:rtype: PVM_tools.bounding_region.BoundingRegion
"""
# this is required so that tracker is re-initialized if it is primed again
self._primed = False
self.im_prev = im
im = cv2.cvtColor(im, cv2.COLOR_RGB2GRAY)
self.cmt.process_frame(im)
self.confidence = 1
if np.isnan(self.cmt.bb).any():
self.bbox = BoundingRegion()
else:
self.bbox = BoundingRegion(image_shape=im.shape, box=self.cmt.bb, confidence=self.confidence)
return self.bbox.copy()
def process(self, heatmap):
"""
The main processing method. Given the probability map, returns a bounding region
:param heatmap: the map (e.g. result of histogram backprojection etc.). Its supposed to be a numpy array of float.
:type heatmap: numpy.ndarray
:return: bounding_box
:rtype: PVM_tools.bounding_region.BoundingRegion
"""
self._last_prob = heatmap
harea = np.prod(heatmap.shape)
if self._last_bb is None or self._last_bb.empty:
(bb, area, mean_prob) = self._find_new_box(heatmap)
confidence = mean_prob
if bb[2] > 0 and bb[3] > 0 and np.prod(bb[2:])<2*harea/3:
self._last_bb = BoundingRegion(image_shape=(heatmap.shape[0], heatmap.shape[1], 3),
box=np.array(bb, dtype=np.int),
confidence=confidence)
else:
# Empty bounding box
self._last_bb = BoundingRegion()
else:
# Step 1 go with the last bbox
_, bb0 = cv2.meanShift(heatmap, tuple(self._last_bb.get_box_pixels()), self.term_crit)
area0 = np.prod(bb0[2:])
mean_prob0 = np.sum(heatmap[bb0[1]:bb0[1] + bb0[3], bb0[0]:bb0[0] + bb0[2]]) / (1e-12 + area0)
_, bb = cv2.CamShift(heatmap, tuple(self._last_bb.get_box_pixels()), self.term_crit)
area = np.prod(bb[2:])
mean_prob = np.sum(heatmap[bb[1]:bb[1] + bb[3], bb[0]:bb[0] + bb[2]]) / (1e-12 + area)
if mean_prob > self.threshold_retrieve and area > self.min_recovered_box_area and area < 2*harea/3:
self._last_bb = BoundingRegion(image_shape=(heatmap.shape[0], heatmap.shape[1], 3),
box=np.array(bb, dtype=np.int),
confidence=mean_prob)
elif mean_prob0 > self.threshold_retrieve and area0 > self.min_recovered_box_area:
# Go with the mean shift box, changing size apparently does no good.
return self._last_bb
else:
# Empty bounding box
self._last_bb = BoundingRegion()
return self._last_bb
def process(self, heatmap):
"""
The main processing method. Given the probability map, returns a bounding region
:param heatmap: the map (e.g. result of histogram backprojection etc.). Its supposed to be a numpy array of float.
:type heatmap: numpy.ndarray
:return: bounding_box
:rtype: PVM_tools.bounding_region.BoundingRegion
"""
self._last_prob = heatmap
harea = np.prod(heatmap.shape)
if self._last_bb is None or self._last_bb.empty:
(bb, area, mean_prob) = self._find_new_box(heatmap)
confidence = mean_prob
if bb[2] > 0 and bb[3] > 0 and np.prod(bb[2:]) < 2 * harea / 3:
self._last_bb = BoundingRegion(image_shape=(heatmap.shape[0],
heatmap.shape[1],
3),
box=np.array(bb, dtype=np.int),
confidence=confidence)
else:
# Empty bounding box
self._last_bb = BoundingRegion()
else:
# Step 1 go with the last bbox
_, bb0 = cv2.meanShift(heatmap,
tuple(self._last_bb.get_box_pixels()),
self.term_crit)
area0 = np.prod(bb0[2:])
mean_prob0 = np.sum(heatmap[bb0[1]:bb0[1] + bb0[3], bb0[0]:bb0[0] +
bb0[2]]) / (1e-12 + area0)
_, bb = cv2.CamShift(heatmap,
tuple(self._last_bb.get_box_pixels()),
self.term_crit)
area = np.prod(bb[2:])
mean_prob = np.sum(heatmap[bb[1]:bb[1] + bb[3],
bb[0]:bb[0] + bb[2]]) / (1e-12 + area)
if mean_prob > self.threshold_retrieve and area > self.min_recovered_box_area and area < 2 * harea / 3:
self._last_bb = BoundingRegion(image_shape=(heatmap.shape[0],
heatmap.shape[1],
3),
box=np.array(bb, dtype=np.int),
confidence=mean_prob)
elif mean_prob0 > self.threshold_retrieve and area0 > self.min_recovered_box_area:
# Go with the mean shift box, changing size apparently does no good.
return self._last_bb
else:
# Empty bounding box
self._last_bb = BoundingRegion()
return self._last_bb
def _track(self, im):
"""
Track on given image, return a bounding box
:param im: image (3 - channel numpy array)
:type im: numpy.ndarray
:return: bounding box of the tracker object
:rtype: PVM_tools.bounding_region.BoundingRegion
"""
# this is required so that tracker is re-initialized if it is primed again
self._primed = False
img_cvmat = cv2.cv.fromarray(im)
self.tld.processImage(img_cvmat)
self.bounding_box = self.tld.getCurrBB()
self.confidence = self.tld.currConf
if self.bounding_box == ():
self.bbox = BoundingRegion()
else:
self.bbox = BoundingRegion(image_shape=(im.shape[0], im.shape[1], 3), box=self.bounding_box, confidence=self.confidence)
return self.bbox.copy()
def reset(self, reload=False):
cv2.destroyAllWindows()
fc = FrameCollection()
if reload:
fc.load_from_file(filename=self.output_file)
else:
fc.load_from_file(filename=self.input_filename)
self.movie = fc
self.tracker = None
self.current_frame_index = 0
self.right_button_pressed = False
self.left_button_pressed = False
self._anchor = None
self.x_size = 30
self.y_size = 30
self._tracked_bounds = None
self._stored_bounds = None
self._current_bounds = BoundingRegion()
# Make sure self.timer is set
self.schedule_callback()
# Initialization of the Graphic User Interface
self.ready_to_refresh = threading.Lock()
self.timer_lock = threading.Lock()
self.win_name = 'Labeling video GUI'
self.image = self.movie.Frame(
self.current_frame_index).get_image(channel=self.channel)
self.display_image = self.movie.Frame(
self.current_frame_index).get_image(channel=self.channel)
self.image_shape = self.image.shape
self.create_windows()
self.set_target_absent()
self.image_buffer = {}
self._last_update = time.time()
self.refresh_timer = threading.Timer(0.05, self.refresh)
self.refresh_timer.start()
self.needs_refresh = True
self.trim_end = len(self.movie) - 1
self.trim_start = 0
def _track(self, im):
"""
Track on given image, return a bounding box
:param im: image (3 - channel numpy array)
:type im: numpy.ndarray
:return: bounding box of the tracker object
:rtype: PVM_tools.bounding_region.BoundingRegion
"""
# this is required so that tracker is re-initialized if it is primed again
self._primed = False
img_cvmat = cv2.cv.fromarray(im)
self.tld.processImage(img_cvmat)
self.bounding_box = self.tld.getCurrBB()
self.confidence = self.tld.currConf
if self.bounding_box == ():
self.bbox = BoundingRegion()
else:
self.bbox = BoundingRegion(image_shape=(im.shape[0], im.shape[1],
3),
box=self.bounding_box,
confidence=self.confidence)
return self.bbox.copy()
def _track(self, im):
"""
Track on given image, return a bounding box
:param im: image (3 - channel numpy array)
:type im: numpy.ndarray
:return: bounding box of the tracker object
:rtype: PVM_tools.bounding_region.BoundingRegion
"""
# this is required so that tracker is re-initialized if it is primed again
self._primed = False
struck.STRUCK_track(im)
struck_bbox = struck.STRUCK_get_bbox()
self.bounding_box = [
struck_bbox["xmin"], struck_bbox["ymin"], struck_bbox["width"],
struck_bbox["height"]
]
if self.bounding_box == ():
self.bbox = BoundingRegion()
else:
self.bbox = BoundingRegion(image_shape=(im.shape[0], im.shape[1],
3),
box=self.bounding_box)
return self.bbox.copy()
def _prime(self, im, bounding_region):
"""
prime tracker on image and bounding box
:param im: input image (3 - channel numpy array)
:type im: numpy.ndarray
:param bounding_region: initial bounding region of the tracked object
:type bounding_region: PVM_tools.bounding_region.BoundingRegion
"""
box = np.array([(im.shape[1]-im.shape[1]*self.size_factor)/2,
(im.shape[0]-im.shape[0]*self.size_factor)/2,
im.shape[1]*self.size_factor,
im.shape[0]*self.size_factor]).astype(np.int)
self.bbox = BoundingRegion(image_shape=im.shape, box=box)
if not self._primed:
self._primed = True
def interpolate(self, start, end):
print "Interpolating %d %d" % (start, end)
label0 = self.movie.Frame(start).get_label(channel=self.channel,
target=self.target)
label1 = self.movie.Frame(end).get_label(channel=self.channel,
target=self.target)
if (label0.empty) or (label1.empty):
return
box0 = label0.get_box_pixels()
box1 = label1.get_box_pixels()
for i in xrange(start + 1, end, 1):
alpha = (end - i) * 1.0 / (end - start)
box2 = alpha * np.array(box0) + (1 - alpha) * np.array(box1)
box2 = map(lambda x: int(x), box2)
self.movie.Frame(i).set_label(BoundingRegion(
box=box2,
image_shape=self.movie.Frame(i).get_image(
channel=self.channel).shape),
channel=self.channel,
target=self.target)
self.completeness[i] = 0
def reset(self, reload=False):
cv2.destroyAllWindows()
fc = FrameCollection()
if reload:
fc.load_from_file(filename=self.output_file)
else:
fc.load_from_file(filename=self.input_filename)
self.movie = fc
self.tracker = None
self.current_frame_index = 0
self.right_button_pressed = False
self.left_button_pressed = False
self._anchor = None
self.x_size = 30
self.y_size = 30
self._tracked_bounds = None
self._stored_bounds = None
self._current_bounds = BoundingRegion()
# Make sure self.timer is set
self.schedule_callback()
# Initialization of the Graphic User Interface
self.ready_to_refresh = threading.Lock()
self.timer_lock = threading.Lock()
self.win_name = 'Labeling video GUI'
self.image = self.movie.Frame(self.current_frame_index).get_image(channel=self.channel)
self.display_image = self.movie.Frame(self.current_frame_index).get_image(channel=self.channel)
self.image_shape = self.image.shape
self.create_windows()
self.set_target_absent()
self.image_buffer = {}
self._last_update = time.time()
self.refresh_timer = threading.Timer(0.05, self.refresh)
self.refresh_timer.start()
self.needs_refresh = True
self.trim_end = len(self.movie)-1
self.trim_start = 0
class TLDVisionTracker(GenericVisionTracker):
"""
This class exposes the open tld tracker implemented in C++ by http://www.gnebehay.com/tld/.
Originally OpenTLD that was originally published in MATLAB by Zdenek Kalal.
"""
def __init__(self):
"""
Initialize the tracker
"""
self.name = 'tld_tracker'
self.reset()
def reset(self):
"""
Reset the tracker
:return:
"""
self.tld = None
self._primed = False
self.bbox = None
def _prime(self, im, bounding_region):
"""
prime tracker on image and bounding box
:param im: input image (3 - channel numpy array)
:type im: numpy.ndarray
:param bounding_region: initial bounding region of the tracked object
:type bounding_region: PVM_tools.bounding_region.BoundingRegion
"""
self.bbox = bounding_region
bounding_box = bounding_region.get_box_pixels()
if not self._primed:
self.tld = tld.TLD2()
self._primed = True
self.height, self.width = im.shape[:2]
self.tld.set_width_and_height((self.width, self.height))
im = cv2.cvtColor(im, cv2.COLOR_RGB2GRAY)
img_cvmat = cv2.cv.fromarray(im)
if bounding_box[0] + bounding_box[2] > self.width:
bounding_box[2] = self.width - bounding_box[0]
if bounding_box[1] + bounding_box[3] > self.height:
bounding_box[3] = self.height - bounding_box[1]
self.tld.selectObject(
img_cvmat,
tuple([
int(bounding_box[0]),
int(bounding_box[1]),
int(bounding_box[2]),
int(bounding_box[3])
]))
def _track(self, im):
"""
Track on given image, return a bounding box
:param im: image (3 - channel numpy array)
:type im: numpy.ndarray
:return: bounding box of the tracker object
:rtype: PVM_tools.bounding_region.BoundingRegion
"""
# this is required so that tracker is re-initialized if it is primed again
self._primed = False
img_cvmat = cv2.cv.fromarray(im)
self.tld.processImage(img_cvmat)
self.bounding_box = self.tld.getCurrBB()
self.confidence = self.tld.currConf
if self.bounding_box == ():
self.bbox = BoundingRegion()
else:
self.bbox = BoundingRegion(image_shape=(im.shape[0], im.shape[1],
3),
box=self.bounding_box,
confidence=self.confidence)
return self.bbox.copy()
def get_heatmap(self, heatmap_name=None):
return self.bbox.get_mask()
def set_target_absent(self):
self._current_bounds = BoundingRegion()
class CMTVisionTracker(GenericVisionTracker):
"""
This class exposes the CMT tracker implemented in http://www.gnebehay.com/CMT/.
"""
def __init__(self):
"""
Initialize the tracker
"""
self.name = 'cmt_tracker'
self.reset()
def reset(self):
"""
Reset the tracker
:return:
"""
self.cmt = None
self._primed = False
self.bbox = None
def _prime(self, im, bounding_region):
"""
prime tracker on image and bounding box
:param im: input image (3 - channel numpy array)
:type im: numpy.ndarray
:param bounding_region: initial bounding region of the tracked object
:type bounding_region: PVM_tools.bounding_region.BoundingRegion
"""
self.bbox=bounding_region
bounding_box = bounding_region.get_box_pixels()
if not self._primed:
this_dir = os.path.dirname(os.path.realpath(__file__))
sys.path.append(this_dir+"/original_cmt/")
from CMT import CMT
self.cmt = CMT()
self._primed = True
self.height, self.width = im.shape[:2]
im = cv2.cvtColor(im, cv2.COLOR_RGB2GRAY)
self.cmt.initialise(im_gray0=im,
tl=tuple([bounding_box[0], bounding_box[1]]),
br=tuple([bounding_box[0]+bounding_box[2], bounding_box[1]+bounding_box[3]]))
self.im_prev = im
def _track(self, im):
"""
Track on given image, return a bounding box
:param im: image (3 - channel numpy array)
:type im: numpy.ndarray
:return: bounding box of the tracker object
:rtype: PVM_tools.bounding_region.BoundingRegion
"""
# this is required so that tracker is re-initialized if it is primed again
self._primed = False
self.im_prev = im
im = cv2.cvtColor(im, cv2.COLOR_RGB2GRAY)
self.cmt.process_frame(im)
self.confidence = 1
if np.isnan(self.cmt.bb).any():
self.bbox = BoundingRegion()
else:
self.bbox = BoundingRegion(image_shape=im.shape, box=self.cmt.bb, confidence=self.confidence)
return self.bbox.copy()
def get_heatmap(self, heatmap_name=None):
return self.bbox.get_mask()
class StruckTracker(GenericVisionTracker):
"""
TODO
"""
def __init__(self):
"""
Initialize the tracker
"""
self.name = 'struck_tracker'
f = open("config.txt", "w")
f.write(struck_config)
f.close()
self.reset()
def reset(self):
"""
Reset the tracker
:return:
"""
self._primed = False
self.bbox = None
def _prime(self, im, bounding_region):
"""
prime tracker on image and bounding box
:param im: input image (3 - channel numpy array)
:type im: numpy.ndarray
:param bounding_region: initial bounding region of the tracked object
:type bounding_region: PVM_tools.bounding_region.BoundingRegion
"""
self.bbox = bounding_region
bounding_box = bounding_region.get_box_pixels()
struck.STRUCK_init(im, bounding_box)
def _track(self, im):
"""
Track on given image, return a bounding box
:param im: image (3 - channel numpy array)
:type im: numpy.ndarray
:return: bounding box of the tracker object
:rtype: PVM_tools.bounding_region.BoundingRegion
"""
# this is required so that tracker is re-initialized if it is primed again
self._primed = False
struck.STRUCK_track(im)
struck_bbox = struck.STRUCK_get_bbox()
self.bounding_box = [
struck_bbox["xmin"], struck_bbox["ymin"], struck_bbox["width"],
struck_bbox["height"]
]
if self.bounding_box == ():
self.bbox = BoundingRegion()
else:
self.bbox = BoundingRegion(image_shape=(im.shape[0], im.shape[1],
3),
box=self.bounding_box)
return self.bbox.copy()
def get_heatmap(self, heatmap_name=None):
return self.bbox.get_mask()
def set_target_absent(self):
self._current_bounds = BoundingRegion()
def _track(self, im):
"""
Track on given image, rseturn a bounding box
:param im: image (3 - channel numpy array)
:type im: numpy.ndarray
:return: bounding box of the tracker object
:rtype: PVM_tools.bounding_region.BoundingRegion
"""
# this is required so that tracker is re-initialized if it is primed again
current_frame = cv2.resize(im, dsize=self.image_size)
self.prop_dict['input_array'][:] = current_frame
self.prop_dict['input_array_float'][:] = current_frame.astype(
np.float) / 255
for i in range(self.step_per_frame):
self.executor.step()
norm = 1.0 / len(self.prop_dict['predicted_readout_arrays'])
self.readout_heatmap[:] = 0
for k in self.prop_dict['predicted_readout_arrays']:
self.readout_heatmap[:] += cv2.resize(k.view(np.ndarray),
dsize=self.image_size) * norm
am = np.unravel_index(np.argmax(self.readout_heatmap),
self.readout_heatmap.shape)
if len(am) == 3:
for d in range(3):
if am[2] != d:
self.readout_heatmap[:, :, d] = 0
self.heatmap = self.readout_heatmap.view(np.ndarray)
if len(self.heatmap.shape) == 3:
self.heatmap = np.max(self.heatmap, axis=2)
self.heatmap = cv2.resize(self.heatmap,
dsize=(im.shape[1], im.shape[0]),
interpolation=cv2.INTER_CUBIC)
self.heatmap = (self.heatmap * 255).astype(np.uint8)
if np.max(self.heatmap) > (np.median(self.heatmap) + self.threshold):
threshold = (np.max(self.heatmap) - np.median(
self.heatmap)) * 0.5 + np.median(self.heatmap)
ret, thresh = cv2.threshold(self.heatmap, threshold, 255, 0)
contours, hierarchy = cv2.findContours(thresh, cv2.RETR_TREE,
cv2.CHAIN_APPROX_SIMPLE)
if len(contours) > 0:
max_cnt = None
for cnt in contours:
pt = np.unravel_index(np.argmax(self.heatmap),
self.heatmap.shape)
if cv2.pointPolygonTest(cnt, (pt[1], pt[0]), False) >= 0:
max_cnt = cnt
break
if max_cnt is None:
self.bbox = BoundingRegion()
else:
x, y, w, h = cv2.boundingRect(max_cnt)
if w * h > 25:
self.bbox = BoundingRegion(image_shape=im.shape,
box=[x, y, w, h])
self.bbox.scale(1.1)
else:
self.bbox = BoundingRegion()
else:
self.bbox = BoundingRegion()
else:
self.bbox = BoundingRegion()
self._primed = False
return self.bbox.copy()
class LabelingApp(object):
def __init__(self,
filename,
output=None,
channel="default",
target="default"):
self.input_filename = filename
if output is None:
self.output_file = self.input_filename
else:
self.output_file = output
self.channel = channel
self.target = target
self.reset()
def reset(self, reload=False):
cv2.destroyAllWindows()
fc = FrameCollection()
if reload:
fc.load_from_file(filename=self.output_file)
else:
fc.load_from_file(filename=self.input_filename)
self.movie = fc
self.tracker = None
self.current_frame_index = 0
self.right_button_pressed = False
self.left_button_pressed = False
self._anchor = None
self.x_size = 30
self.y_size = 30
self._tracked_bounds = None
self._stored_bounds = None
self._current_bounds = BoundingRegion()
# Make sure self.timer is set
self.schedule_callback()
# Initialization of the Graphic User Interface
self.ready_to_refresh = threading.Lock()
self.timer_lock = threading.Lock()
self.win_name = 'Labeling video GUI'
self.image = self.movie.Frame(
self.current_frame_index).get_image(channel=self.channel)
self.display_image = self.movie.Frame(
self.current_frame_index).get_image(channel=self.channel)
self.image_shape = self.image.shape
self.create_windows()
self.set_target_absent()
self.image_buffer = {}
self._last_update = time.time()
self.refresh_timer = threading.Timer(0.05, self.refresh)
self.refresh_timer.start()
self.needs_refresh = True
self.trim_end = len(self.movie) - 1
self.trim_start = 0
def fill_up_the_buffer(self):
r0 = max(self.current_frame_index - 5, 0)
r1 = min(self.current_frame_index + 5, len(self.movie))
for i in xrange(r0, r1, 1):
if i not in self.image_buffer.keys():
self.image_buffer[i] = self.movie.Frame(i).get_image(
channel=self.channel)
for i in self.image_buffer.keys():
if i < r0 or i > r1:
del self.image_buffer[i]
def create_windows(self):
''' Create playback progress bar (allow user to move along the movie) '''
cv2.namedWindow(self.win_name)
cv2.moveWindow(self.win_name, 150, 150)
cv2.namedWindow("completeness")
cv2.moveWindow("completeness", 150, 50)
cv2.setMouseCallback(self.win_name, self.trackbar_window_onMouse)
cv2.createTrackbar("Frame", self.win_name, 0,
len(self.movie) - 1, self.jump_to_frame_callback)
cv2.createTrackbar("Trim start", self.win_name, 0,
len(self.movie) - 1, self.set_trim_start)
cv2.createTrackbar("Trim end", self.win_name,
len(self.movie) - 1,
len(self.movie) - 1, self.set_trim_end)
im_height = 30 # 30 pixels high completeness bar
im_width = len(self.movie)
self.completeness_image = np.zeros((im_height, im_width, 3))
current_frame_index = int(im_width * float(self.current_frame_index) /
len(self.movie))
self.completeness_image[:, current_frame_index, :] = 1.
self.completeness = np.zeros(im_width, dtype=np.uint8)
for i in xrange(len(self.movie)):
bounds = self.movie.Frame(i).get_label(channel=self.channel,
target=self.target)
if bounds is None or bounds.empty:
if bounds is None:
self.movie.Frame(i).set_label(channel=self.channel,
target=self.target,
label=BoundingRegion())
self.completeness[i] = 2
else:
if bounds.is_keyframe():
self.completeness[i] = 1
else:
self.completeness[i] = 0
def set_trim_start(self, frame_index):
self.trim_start = frame_index + 0
def set_trim_end(self, frame_index):
self.trim_end = frame_index + 0
def jump_to_frame_callback(self, frame_index):
self.current_frame_index = frame_index
self.image = self.movie.Frame(
self.current_frame_index).get_image(channel=self.channel)
self.needs_refresh = True
def update_completeness_window(self):
if self.ready_to_refresh.acquire(False):
self.completeness_image *= 0
self.completeness_image[:, self.current_frame_index, :] = 255
for i in xrange(len(self.movie)):
self.completeness_image[:, i, self.completeness[i]] = 255
cv2.putText(self.completeness_image,
'%i' % self.current_frame_index, (10, 10),
cv2.FONT_HERSHEY_PLAIN,
1.0, (255, 255, 255),
lineType=cv2.CV_AA)
cv2.imshow('completeness', self.completeness_image)
self.ready_to_refresh.release()
def update_trackbar_window(self):
if self.ready_to_refresh.acquire():
image = self.image.copy()
self._stored_bounds = self.movie.Frame(
self.current_frame_index).get_label(channel=self.channel,
target=self.target)
# display rectangle over selected area
self._current_bounds.draw_box(image, color=(255, 255, 255))
self._current_bounds.draw_box(image, color=(0, 0, 0), thickness=1)
# display rectangle over stored area
if self._stored_bounds is not None:
self._stored_bounds.draw_box(image, color=(255, 0, 0))
if self._tracked_bounds is not None:
self._tracked_bounds.draw_box(image, color=(0, 255, 0))
# reshape to display if too small
min_width = 320
if image.shape[1] < min_width:
resize_height = int(0.5 + image.shape[0] * min_width /
float(image.shape[1]))
image = cv2.resize(image, (min_width, resize_height),
interpolation=cv2.INTER_NEAREST)
self.display_image = image
self.ready_to_refresh.release()
def refresh(self):
if self.timer_lock.acquire():
if self.needs_refresh:
self.update_trackbar_window()
self.update_completeness_window()
self.needs_refresh = False
self.refresh_timer = threading.Timer(0.05, self.refresh)
self.refresh_timer.start()
self.timer_lock.release()
def trackbar_window_onMouse(self, event, x, y, flags, _):
cv2.imshow(self.win_name, self.display_image)
# Update state
self.right_button_pressed = (flags & cv2.EVENT_FLAG_RBUTTON
) != 0 and event != cv2.EVENT_RBUTTONUP
self.left_button_pressed = (flags & cv2.EVENT_FLAG_LBUTTON
) != 0 and event != cv2.EVENT_LBUTTONUP
# Set bounding box
if event == cv2.EVENT_MBUTTONDOWN:
if self._anchor is None:
# Setting the anchor
self._anchor = (x, y)
else:
# Defining the box
self.save_and_advance()
elif self._anchor is not None and event != cv2.EVENT_MBUTTONUP:
# We are creating new bounding box from anchor to current position
self.x_size = max(5, int(abs(x - self._anchor[0]) + 0.5))
self.y_size = max(5, int(abs(y - self._anchor[1]) + 0.5))
center_x = (x + self._anchor[0]) / 2
center_y = (y + self._anchor[1]) / 2
self.set_bounding_box_with_center(center_x, center_y)
elif event == cv2.EVENT_MBUTTONUP:
self.x_size = max(5, int(abs(x - self._anchor[0]) + 0.5))
self.y_size = max(5, int(abs(y - self._anchor[1]) + 0.5))
center_x = (x + self._anchor[0]) / 2
center_y = (y + self._anchor[1]) / 2
self.set_bounding_box_with_center(center_x, center_y)
keyframe = False
if flags & cv2.EVENT_FLAG_CTRLKEY != 0:
keyframe = True
self.save_and_advance(keyframe=keyframe)
else:
self.set_bounding_box_with_center(x, y)
# If this is a left/right button down event,
# we can advance one frame immediately
# and then start the timer
if event == cv2.EVENT_RBUTTONDOWN:
self.set_target_absent()
self.save_and_advance()
self.right_button_pressed = True
if self.timer.finished:
# Rearm the timer
self.schedule_callback()
elif event == cv2.EVENT_LBUTTONDOWN:
self.set_bounding_box_with_center(x, y)
keyframe = False
if flags & cv2.EVENT_FLAG_CTRLKEY != 0:
keyframe = True
self.save_and_advance(keyframe=keyframe)
self.left_button_pressed = True
if self.timer.finished:
# Rearm the timer
self.schedule_callback()
elif not (self.right_button_pressed or self.left_button_pressed):
# No mouse button pressed, so cancel the timer if it's running
self.timer.cancel()
if time.time() - self._last_update > 0.1:
self._last_update = time.time()
self.needs_refresh = True
def set_target_absent(self):
self._current_bounds = BoundingRegion()
def set_bounding_box_with_center(self, x, y):
box = [
np.clip(int(x + 0.5) - self.x_size / 2, 0, self.image_shape[1]),
np.clip(int(y + 0.5) - self.y_size / 2, 0, self.image_shape[0]),
self.x_size + min(int(x + 0.5) - self.x_size / 2, 0),
self.y_size + min(int(y + 0.5) - self.y_size / 2, 0)
]
self._current_bounds = BoundingRegion(box=box)
self.needs_refresh = True
def schedule_callback(self):
# 0.2 secs => 5 Hz
self.timer = threading.Timer(0.1, self.timer_callback)
self.timer.start()
def timer_callback(self):
if self.right_button_pressed or self.left_button_pressed:
self.save_and_advance()
self.schedule_callback()
def advance_current_frame(self, increment=1):
self.current_frame_index = min(
len(self.movie) - 1, max(0, self.current_frame_index + increment))
self.image = self.movie.Frame(
self.current_frame_index).get_image(channel=self.channel)
cv2.setTrackbarPos("Frame", self.win_name, self.current_frame_index)
if self.tracker is not None:
self._tracked_bounds = self.tracker.track(
self.movie.Frame(
self.current_frame_index).get_image(channel=self.channel))
def save_and_advance(self, keyframe=False):
self._anchor = None
self.movie.Frame(self.current_frame_index).set_label(
self._current_bounds.copy(),
channel=self.channel,
target=self.target)
if self._current_bounds.empty:
self.completeness[self.current_frame_index] = 2
else:
if self._current_bounds.is_keyframe():
self.completeness[self.current_frame_index] = 1
else:
self.completeness[self.current_frame_index] = 0
if keyframe:
self.make_keyframe()
self.advance_current_frame(15)
else:
self.advance_current_frame(1)
def toggle_tracker(self):
if self.tracker is None:
self.tracker = CMTVisionTracker()
self.tracker.prime(
self.movie.Frame(
self.current_frame_index).get_image(channel=self.channel),
self._current_bounds)
logging.warning("Tracker is enabled")
else:
self.tracker = None
self._tracked_bounds = None
logging.warning("Tracker is disabled")
def advance_tracker(self):
if self._tracked_bounds is not None:
self._current_bounds = self._tracked_bounds
self.save_and_advance()
def export_movie(self):
trim_frames_front = self.trim_start
trim_frames_end = (len(self.movie) - 1) - self.trim_end
if trim_frames_end > 0:
for i in range(trim_frames_end):
self.movie.delete(-1)
if trim_frames_front > 0:
for i in range(trim_frames_front):
self.movie.delete(0)
logging.warning("Exporting result in file " + self.output_file)
self.movie.write_to_file(self.output_file)
logging.warning("Exporting completed !")
def interpolate(self, start, end):
print "Interpolating %d %d" % (start, end)
label0 = self.movie.Frame(start).get_label(channel=self.channel,
target=self.target)
label1 = self.movie.Frame(end).get_label(channel=self.channel,
target=self.target)
if (label0.empty) or (label1.empty):
return
box0 = label0.get_box_pixels()
box1 = label1.get_box_pixels()
for i in xrange(start + 1, end, 1):
alpha = (end - i) * 1.0 / (end - start)
box2 = alpha * np.array(box0) + (1 - alpha) * np.array(box1)
box2 = map(lambda x: int(x), box2)
self.movie.Frame(i).set_label(BoundingRegion(
box=box2,
image_shape=self.movie.Frame(i).get_image(
channel=self.channel).shape),
channel=self.channel,
target=self.target)
self.completeness[i] = 0
def make_keyframe(self):
self.completeness[self.current_frame_index] = 1
self.movie.Frame(self.current_frame_index).get_label(
channel=self.channel, target=self.target).set_keyframe(True)
# find previous keyframe
next_keyframe = -1
for i in xrange(self.current_frame_index + 1, len(self.movie), 1):
bounds = self.movie.Frame(i).get_label(channel=self.channel,
target=self.target)
if bounds.is_keyframe():
next_keyframe = i
break
if next_keyframe >= 0:
print next_keyframe
self.interpolate(self.current_frame_index, next_keyframe)
# find next keyframe
prev_keyframe = -1
for i in xrange(self.current_frame_index - 1, -1, -1):
bounds = self.movie.Frame(i).get_label(channel=self.channel,
target=self.target)
if bounds.is_keyframe():
prev_keyframe = i
break
if prev_keyframe >= 0:
print prev_keyframe
self.interpolate(prev_keyframe, self.current_frame_index)
def run(self):
while True:
key = cv2.waitKey(0)
cv2.imshow(self.win_name, self.display_image)
if key == 65361: # left arrow
# Go to previous frame
self.advance_current_frame(-1)
elif key == 65362:
self.advance_current_frame(-10)
elif key == 65363: # right arrow
# Go to next frame
self.advance_current_frame()
elif key == 65364:
self.advance_current_frame(10)
elif key == ord('w'): # Write
self.export_movie()
if self.timer_lock.acquire():
if not self.refresh_timer.finished:
self.refresh_timer.cancel()
self.timer_lock.release()
self.timer.cancel()
self.reset()
elif key == ord('t'): # Tracker
self.toggle_tracker()
elif key == ord(' '): # Advance tracker
self.advance_tracker()
self.needs_refresh = True
elif key == ord('a'):
self.x_size = int(self.x_size * 1.1)
self.y_size = int(self.y_size * 1.1)
self.needs_refresh = True
elif key == ord('z'):
self.x_size = int(self.x_size * 1 / 1.1)
self.y_size = int(self.y_size * 1 / 1.1)
self.needs_refresh = True
elif key == ord('s'):
self.y_size = int(self.y_size * 1.1)
self.needs_refresh = True
elif key == ord('k'):
self.make_keyframe()
elif key == ord('x'):
self.y_size = int(self.y_size * 1 / 1.1)
self.needs_refresh = True
elif key == ord('q'): # Export
while key != -1:
for k in range(10):
key = cv2.waitKey(10)
print "Quiting! Do you want to save your work? [Y/N]"
while True:
key = cv2.waitKey(0)
if key == ord('Y') or key == ord('y'):
self.export_movie()
break
if key == ord('N') or key == ord('n'):
break
break
logging.warning('Exiting')
self.refresh_timer.cancel()
class TLDVisionTracker(GenericVisionTracker):
"""
This class exposes the open tld tracker implemented in C++ by http://www.gnebehay.com/tld/.
Originally OpenTLD that was originally published in MATLAB by Zdenek Kalal.
"""
def __init__(self):
"""
Initialize the tracker
"""
self.name = 'tld_tracker'
self.reset()
def reset(self):
"""
Reset the tracker
:return:
"""
self.tld = None
self._primed = False
self.bbox = None
def _prime(self, im, bounding_region):
"""
prime tracker on image and bounding box
:param im: input image (3 - channel numpy array)
:type im: numpy.ndarray
:param bounding_region: initial bounding region of the tracked object
:type bounding_region: PVM_tools.bounding_region.BoundingRegion
"""
self.bbox=bounding_region
bounding_box = bounding_region.get_box_pixels()
if not self._primed:
self.tld = tld.TLD2()
self._primed = True
self.height, self.width = im.shape[:2]
self.tld.set_width_and_height((self.width, self.height))
im = cv2.cvtColor(im, cv2.COLOR_RGB2GRAY)
img_cvmat = cv2.cv.fromarray(im)
if bounding_box[0] + bounding_box[2] > self.width:
bounding_box[2] = self.width - bounding_box[0]
if bounding_box[1] + bounding_box[3] > self.height:
bounding_box[3] = self.height - bounding_box[1]
self.tld.selectObject(img_cvmat, tuple([int(bounding_box[0]), int(bounding_box[1]), int(bounding_box[2]), int(bounding_box[3])]))
def _track(self, im):
"""
Track on given image, return a bounding box
:param im: image (3 - channel numpy array)
:type im: numpy.ndarray
:return: bounding box of the tracker object
:rtype: PVM_tools.bounding_region.BoundingRegion
"""
# this is required so that tracker is re-initialized if it is primed again
self._primed = False
img_cvmat = cv2.cv.fromarray(im)
self.tld.processImage(img_cvmat)
self.bounding_box = self.tld.getCurrBB()
self.confidence = self.tld.currConf
if self.bounding_box == ():
self.bbox = BoundingRegion()
else:
self.bbox = BoundingRegion(image_shape=(im.shape[0], im.shape[1], 3), box=self.bounding_box, confidence=self.confidence)
return self.bbox.copy()
def get_heatmap(self, heatmap_name=None):
return self.bbox.get_mask()
class CamshiftBoundingBoxer(AbstractBoundingBoxer):
"""
Given a prob-map of pixel membership, return a bounding box: (PVM_tools.bounding_region.BoundingRegion)
This class uses the opencv CamShift method to estimate the new position of the bounding box
based on the previous one. In the case the previous bounding box is empty, then a search over the
entire map is performed to estimate the new position.
"""
def __init__(self, recovery_kernel_size=20, min_recovered_box_area=400, threshold_retrieve=0.2):
"""
:param recovery_kernel_size: when searching for a new box, the probablity map will be filtered with a box filter, to smooth out single peaks and find a better candidate. This is the size of the box kernal.
:type recovery_kernel_size: int
:param min_recovered_box_area: after finding a box, assert that width x height is at least this number of pixels. Otherwise do not accept that as a good box and return an empty box.
:type min_recovered_box_area: int
:param threshold_retrieve: When object the is lost, that is the minimal confidence the tracker must get to consider that a new candidate is considered as the new target Lower value (ex 0.1) provides better recovery but more false positive. Typical value are between 0.1 and 0.2
:type threshold_retrieve: float
"""
self.term_crit = (cv2.TERM_CRITERIA_EPS | cv2.TERM_CRITERIA_COUNT, 10, 1)
self._last_bb = None
self.confidence = 1.
self.recovery_kernel_size = recovery_kernel_size
self.min_recovered_box_area = min_recovered_box_area
self.threshold_retrieve = threshold_retrieve
def reset(self):
"""
Reset the object.
"""
self._last_bb = None
def _find_new_box(self, heatmap):
heatmap = cv2.boxFilter(heatmap, ddepth=-1, ksize=(self.recovery_kernel_size, self.recovery_kernel_size), borderType=cv2.BORDER_REPLICATE)
# new_bb = np.array(np.unravel_index(np.argmax(conv_prob), conv_prob.shape))[::-1]
# new_bb = np.array(tuple(new_bb - max(1, min(10, self.recovery_kernel_size/2))) + (min(20, self.recovery_kernel_size), ) * 2)
# _, bb_reset = cv2.CamShift(heatmap, tuple(new_bb), self.term_crit)
peak = np.unravel_index(np.argmax(heatmap), heatmap.shape)
_, heatmap = cv2.threshold(heatmap, np.max(heatmap)*0.9, 255, cv2.THRESH_BINARY)
_, bb_reset = cv2.floodFill(heatmap, None, tuple(peak[::-1]), 255, loDiff=10, flags=cv2.FLOODFILL_FIXED_RANGE)
bb_area = bb_reset[2] * bb_reset[3]
mean_prob_reset = np.sum(heatmap[bb_reset[1]:bb_reset[1] + bb_reset[3], bb_reset[0]:bb_reset[0] + bb_reset[2]]) / float(bb_area)
return bb_reset, bb_area, mean_prob_reset
def process(self, heatmap):
"""
The main processing method. Given the probability map, returns a bounding region
:param heatmap: the map (e.g. result of histogram backprojection etc.). Its supposed to be a numpy array of float.
:type heatmap: numpy.ndarray
:return: bounding_box
:rtype: PVM_tools.bounding_region.BoundingRegion
"""
self._last_prob = heatmap
harea = np.prod(heatmap.shape)
if self._last_bb is None or self._last_bb.empty:
(bb, area, mean_prob) = self._find_new_box(heatmap)
confidence = mean_prob
if bb[2] > 0 and bb[3] > 0 and np.prod(bb[2:])<2*harea/3:
self._last_bb = BoundingRegion(image_shape=(heatmap.shape[0], heatmap.shape[1], 3),
box=np.array(bb, dtype=np.int),
confidence=confidence)
else:
# Empty bounding box
self._last_bb = BoundingRegion()
else:
# Step 1 go with the last bbox
_, bb0 = cv2.meanShift(heatmap, tuple(self._last_bb.get_box_pixels()), self.term_crit)
area0 = np.prod(bb0[2:])
mean_prob0 = np.sum(heatmap[bb0[1]:bb0[1] + bb0[3], bb0[0]:bb0[0] + bb0[2]]) / (1e-12 + area0)
_, bb = cv2.CamShift(heatmap, tuple(self._last_bb.get_box_pixels()), self.term_crit)
area = np.prod(bb[2:])
mean_prob = np.sum(heatmap[bb[1]:bb[1] + bb[3], bb[0]:bb[0] + bb[2]]) / (1e-12 + area)
if mean_prob > self.threshold_retrieve and area > self.min_recovered_box_area and area < 2*harea/3:
self._last_bb = BoundingRegion(image_shape=(heatmap.shape[0], heatmap.shape[1], 3),
box=np.array(bb, dtype=np.int),
confidence=mean_prob)
elif mean_prob0 > self.threshold_retrieve and area0 > self.min_recovered_box_area:
# Go with the mean shift box, changing size apparently does no good.
return self._last_bb
else:
# Empty bounding box
self._last_bb = BoundingRegion()
return self._last_bb
def set_current_bounding_box(self, bounding_box):
"""
If the bounding box is somehow available (e.g. in priming) that values can be passed in here to affect
future computation (new boxes will be found in reference to this one)
:param bounding_box: A bounding region object
:type bounding_box: PVM_tools.bounding_region.BoundingRegion
"""
self._last_bb = bounding_box
class PVMVisionTracker(GenericVisionTracker):
"""
This class exposes the null vision tracker which just always
returns its priming bounding box
"""
def __init__(self,
filename="",
remote_filename="",
cores="4",
storage=None,
steps_per_frame=1):
"""
Initialize the tracker
"""
self.name = 'PVMtracker'
if filename == "":
filename = storage.get(remote_filename)
self.prop_dict = CoreUtils.load_model(filename)
logging.info("Loaded the dictionary %s", filename)
PVM_Create.upgrade_dictionary_to_ver1_0(self.prop_dict)
self.prop_dict['num_proc'] = int(cores)
for k in range(len(self.prop_dict['learning_rates'])):
self.prop_dict['learning_rates'][k][0] = 0.0
logging.info("Setting learning rate in layer %d to zero" % k)
self.prop_dict["readout_learning_rate"][0] = 0.0
logging.info("Setting readout learning rate to zero")
self.manager = Manager(self.prop_dict, 1000)
self.executor = CoreUtils.ModelExecution(prop_dict=self.prop_dict,
manager=self.manager,
port=9100)
self.executor.start(blocking=False)
self.threshold = 32
self.image_size = self.prop_dict['input_array'].shape[:2][::-1]
self.readout_heatmap = np.zeros(self.image_size, dtype=np.float)
self.step_per_frame = steps_per_frame
def reset(self):
"""
Reset the tracker
:return:
"""
self._primed = False
self.bbox = None
def _prime(self, im, bounding_region):
"""
prime tracker on image and bounding box
:param im: input image (3 - channel numpy array)
:type im: numpy.ndarray
:param bounding_region: initial bounding region of the tracked object
:type bounding_region: PVM_tools.bounding_region.BoundingRegion
"""
if not self._primed:
logging.info("Priming the PVM tracker")
logging.info("Setting up the tracker threshold to %d" %
self.threshold)
self._primed = True
def _track(self, im):
"""
Track on given image, rseturn a bounding box
:param im: image (3 - channel numpy array)
:type im: numpy.ndarray
:return: bounding box of the tracker object
:rtype: PVM_tools.bounding_region.BoundingRegion
"""
# this is required so that tracker is re-initialized if it is primed again
current_frame = cv2.resize(im, dsize=self.image_size)
self.prop_dict['input_array'][:] = current_frame
self.prop_dict['input_array_float'][:] = current_frame.astype(
np.float) / 255
for i in range(self.step_per_frame):
self.executor.step()
norm = 1.0 / len(self.prop_dict['predicted_readout_arrays'])
self.readout_heatmap[:] = 0
for k in self.prop_dict['predicted_readout_arrays']:
self.readout_heatmap[:] += cv2.resize(k.view(np.ndarray),
dsize=self.image_size) * norm
am = np.unravel_index(np.argmax(self.readout_heatmap),
self.readout_heatmap.shape)
if len(am) == 3:
for d in range(3):
if am[2] != d:
self.readout_heatmap[:, :, d] = 0
self.heatmap = self.readout_heatmap.view(np.ndarray)
if len(self.heatmap.shape) == 3:
self.heatmap = np.max(self.heatmap, axis=2)
self.heatmap = cv2.resize(self.heatmap,
dsize=(im.shape[1], im.shape[0]),
interpolation=cv2.INTER_CUBIC)
self.heatmap = (self.heatmap * 255).astype(np.uint8)
if np.max(self.heatmap) > (np.median(self.heatmap) + self.threshold):
threshold = (np.max(self.heatmap) - np.median(
self.heatmap)) * 0.5 + np.median(self.heatmap)
ret, thresh = cv2.threshold(self.heatmap, threshold, 255, 0)
contours, hierarchy = cv2.findContours(thresh, cv2.RETR_TREE,
cv2.CHAIN_APPROX_SIMPLE)
if len(contours) > 0:
max_cnt = None
for cnt in contours:
pt = np.unravel_index(np.argmax(self.heatmap),
self.heatmap.shape)
if cv2.pointPolygonTest(cnt, (pt[1], pt[0]), False) >= 0:
max_cnt = cnt
break
if max_cnt is None:
self.bbox = BoundingRegion()
else:
x, y, w, h = cv2.boundingRect(max_cnt)
if w * h > 25:
self.bbox = BoundingRegion(image_shape=im.shape,
box=[x, y, w, h])
self.bbox.scale(1.1)
else:
self.bbox = BoundingRegion()
else:
self.bbox = BoundingRegion()
else:
self.bbox = BoundingRegion()
self._primed = False
return self.bbox.copy()
def get_heatmap(self, heatmap_name=None):
return self.heatmap()
def finish(self):
self.manager._running = False
self.executor.step()
self.executor.finish()
class CamshiftBoundingBoxer(AbstractBoundingBoxer):
"""
Given a prob-map of pixel membership, return a bounding box: (PVM_tools.bounding_region.BoundingRegion)
This class uses the opencv CamShift method to estimate the new position of the bounding box
based on the previous one. In the case the previous bounding box is empty, then a search over the
entire map is performed to estimate the new position.
"""
def __init__(self,
recovery_kernel_size=20,
min_recovered_box_area=400,
threshold_retrieve=0.2):
"""
:param recovery_kernel_size: when searching for a new box, the probablity map will be filtered with a box filter, to smooth out single peaks and find a better candidate. This is the size of the box kernal.
:type recovery_kernel_size: int
:param min_recovered_box_area: after finding a box, assert that width x height is at least this number of pixels. Otherwise do not accept that as a good box and return an empty box.
:type min_recovered_box_area: int
:param threshold_retrieve: When object the is lost, that is the minimal confidence the tracker must get to consider that a new candidate is considered as the new target Lower value (ex 0.1) provides better recovery but more false positive. Typical value are between 0.1 and 0.2
:type threshold_retrieve: float
"""
self.term_crit = (cv2.TERM_CRITERIA_EPS | cv2.TERM_CRITERIA_COUNT, 10,
1)
self._last_bb = None
self.confidence = 1.
self.recovery_kernel_size = recovery_kernel_size
self.min_recovered_box_area = min_recovered_box_area
self.threshold_retrieve = threshold_retrieve
def reset(self):
"""
Reset the object.
"""
self._last_bb = None
def _find_new_box(self, heatmap):
heatmap = cv2.boxFilter(heatmap,
ddepth=-1,
ksize=(self.recovery_kernel_size,
self.recovery_kernel_size),
borderType=cv2.BORDER_REPLICATE)
# new_bb = np.array(np.unravel_index(np.argmax(conv_prob), conv_prob.shape))[::-1]
# new_bb = np.array(tuple(new_bb - max(1, min(10, self.recovery_kernel_size/2))) + (min(20, self.recovery_kernel_size), ) * 2)
# _, bb_reset = cv2.CamShift(heatmap, tuple(new_bb), self.term_crit)
peak = np.unravel_index(np.argmax(heatmap), heatmap.shape)
_, heatmap = cv2.threshold(heatmap,
np.max(heatmap) * 0.9, 255,
cv2.THRESH_BINARY)
_, bb_reset = cv2.floodFill(heatmap,
None,
tuple(peak[::-1]),
255,
loDiff=10,
flags=cv2.FLOODFILL_FIXED_RANGE)
bb_area = bb_reset[2] * bb_reset[3]
mean_prob_reset = np.sum(
heatmap[bb_reset[1]:bb_reset[1] + bb_reset[3],
bb_reset[0]:bb_reset[0] + bb_reset[2]]) / float(bb_area)
return bb_reset, bb_area, mean_prob_reset
def process(self, heatmap):
"""
The main processing method. Given the probability map, returns a bounding region
:param heatmap: the map (e.g. result of histogram backprojection etc.). Its supposed to be a numpy array of float.
:type heatmap: numpy.ndarray
:return: bounding_box
:rtype: PVM_tools.bounding_region.BoundingRegion
"""
self._last_prob = heatmap
harea = np.prod(heatmap.shape)
if self._last_bb is None or self._last_bb.empty:
(bb, area, mean_prob) = self._find_new_box(heatmap)
confidence = mean_prob
if bb[2] > 0 and bb[3] > 0 and np.prod(bb[2:]) < 2 * harea / 3:
self._last_bb = BoundingRegion(image_shape=(heatmap.shape[0],
heatmap.shape[1],
3),
box=np.array(bb, dtype=np.int),
confidence=confidence)
else:
# Empty bounding box
self._last_bb = BoundingRegion()
else:
# Step 1 go with the last bbox
_, bb0 = cv2.meanShift(heatmap,
tuple(self._last_bb.get_box_pixels()),
self.term_crit)
area0 = np.prod(bb0[2:])
mean_prob0 = np.sum(heatmap[bb0[1]:bb0[1] + bb0[3], bb0[0]:bb0[0] +
bb0[2]]) / (1e-12 + area0)
_, bb = cv2.CamShift(heatmap,
tuple(self._last_bb.get_box_pixels()),
self.term_crit)
area = np.prod(bb[2:])
mean_prob = np.sum(heatmap[bb[1]:bb[1] + bb[3],
bb[0]:bb[0] + bb[2]]) / (1e-12 + area)
if mean_prob > self.threshold_retrieve and area > self.min_recovered_box_area and area < 2 * harea / 3:
self._last_bb = BoundingRegion(image_shape=(heatmap.shape[0],
heatmap.shape[1],
3),
box=np.array(bb, dtype=np.int),
confidence=mean_prob)
elif mean_prob0 > self.threshold_retrieve and area0 > self.min_recovered_box_area:
# Go with the mean shift box, changing size apparently does no good.
return self._last_bb
else:
# Empty bounding box
self._last_bb = BoundingRegion()
return self._last_bb
def set_current_bounding_box(self, bounding_box):
"""
If the bounding box is somehow available (e.g. in priming) that values can be passed in here to affect
future computation (new boxes will be found in reference to this one)
:param bounding_box: A bounding region object
:type bounding_box: PVM_tools.bounding_region.BoundingRegion
"""
self._last_bb = bounding_box
class PVMVisionTracker(GenericVisionTracker):
"""
This class exposes the null vision tracker which just always
returns its priming bounding box
"""
def __init__(self, filename="", remote_filename="", cores="4", storage=None, steps_per_frame=1):
"""
Initialize the tracker
"""
self.name = 'PVMtracker'
if filename == "":
filename = storage.get(remote_filename)
self.prop_dict = CoreUtils.load_model(filename)
logging.info("Loaded the dictionary %s", filename)
PVM_Create.upgrade_dictionary_to_ver1_0(self.prop_dict)
self.prop_dict['num_proc'] = int(cores)
for k in range(len(self.prop_dict['learning_rates'])):
self.prop_dict['learning_rates'][k][0] = 0.0
logging.info("Setting learning rate in layer %d to zero" % k)
self.prop_dict["readout_learning_rate"][0] = 0.0
logging.info("Setting readout learning rate to zero")
self.manager = Manager(self.prop_dict, 1000)
self.executor = CoreUtils.ModelExecution(prop_dict=self.prop_dict, manager=self.manager, port=9100)
self.executor.start(blocking=False)
self.threshold = 32
self.image_size = self.prop_dict['input_array'].shape[:2][::-1]
self.readout_heatmap = np.zeros(self.image_size, dtype=np.float)
self.step_per_frame = steps_per_frame
def reset(self):
"""
Reset the tracker
:return:
"""
self._primed = False
self.bbox = None
def _prime(self, im, bounding_region):
"""
prime tracker on image and bounding box
:param im: input image (3 - channel numpy array)
:type im: numpy.ndarray
:param bounding_region: initial bounding region of the tracked object
:type bounding_region: PVM_tools.bounding_region.BoundingRegion
"""
if not self._primed:
logging.info("Priming the PVM tracker")
logging.info("Setting up the tracker threshold to %d" % self.threshold)
self._primed = True
def _track(self, im):
"""
Track on given image, rseturn a bounding box
:param im: image (3 - channel numpy array)
:type im: numpy.ndarray
:return: bounding box of the tracker object
:rtype: PVM_tools.bounding_region.BoundingRegion
"""
# this is required so that tracker is re-initialized if it is primed again
current_frame = cv2.resize(im, dsize=self.image_size)
self.prop_dict['input_array'][:] = current_frame
self.prop_dict['input_array_float'][:] = current_frame.astype(np.float)/255
for i in range(self.step_per_frame):
self.executor.step()
norm = 1.0 / len(self.prop_dict['predicted_readout_arrays'])
self.readout_heatmap[:] = 0
for k in self.prop_dict['predicted_readout_arrays']:
self.readout_heatmap[:] += cv2.resize(k.view(np.ndarray), dsize=self.image_size) * norm
am = np.unravel_index(np.argmax(self.readout_heatmap), self.readout_heatmap.shape)
if len(am) == 3:
for d in range(3):
if am[2] != d:
self.readout_heatmap[:, :, d] = 0
self.heatmap = self.readout_heatmap.view(np.ndarray)
if len(self.heatmap.shape) == 3:
self.heatmap = np.max(self.heatmap, axis=2)
self.heatmap = cv2.resize(self.heatmap, dsize=(im.shape[1], im.shape[0]), interpolation=cv2.INTER_CUBIC)
self.heatmap = (self.heatmap * 255).astype(np.uint8)
if np.max(self.heatmap) > (np.median(self.heatmap)+self.threshold):
threshold=(np.max(self.heatmap)-np.median(self.heatmap))*0.5+np.median(self.heatmap)
ret, thresh = cv2.threshold(self.heatmap, threshold, 255, 0)
contours, hierarchy = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
if len(contours) > 0:
max_cnt=None
for cnt in contours:
pt = np.unravel_index(np.argmax(self.heatmap), self.heatmap.shape)
if cv2.pointPolygonTest(cnt, (pt[1], pt[0]), False)>=0:
max_cnt = cnt
break
if max_cnt is None:
self.bbox = BoundingRegion()
else:
x, y, w, h = cv2.boundingRect(max_cnt)
if w*h > 25:
self.bbox = BoundingRegion(image_shape=im.shape, box=[x, y, w, h])
self.bbox.scale(1.1)
else:
self.bbox = BoundingRegion()
else:
self.bbox = BoundingRegion()
else:
self.bbox = BoundingRegion()
self._primed = False
return self.bbox.copy()
def get_heatmap(self, heatmap_name=None):
return self.heatmap()
def finish(self):
self.manager._running = False
self.executor.step()
self.executor.finish()
class LabelingApp(object):
def __init__(self, filename, output=None, channel="default", target="default"):
self.input_filename = filename
if output is None:
self.output_file = self.input_filename
else:
self.output_file = output
self.channel = channel
self.target = target
self.reset()
def reset(self, reload=False):
cv2.destroyAllWindows()
fc = FrameCollection()
if reload:
fc.load_from_file(filename=self.output_file)
else:
fc.load_from_file(filename=self.input_filename)
self.movie = fc
self.tracker = None
self.current_frame_index = 0
self.right_button_pressed = False
self.left_button_pressed = False
self._anchor = None
self.x_size = 30
self.y_size = 30
self._tracked_bounds = None
self._stored_bounds = None
self._current_bounds = BoundingRegion()
# Make sure self.timer is set
self.schedule_callback()
# Initialization of the Graphic User Interface
self.ready_to_refresh = threading.Lock()
self.timer_lock = threading.Lock()
self.win_name = 'Labeling video GUI'
self.image = self.movie.Frame(self.current_frame_index).get_image(channel=self.channel)
self.display_image = self.movie.Frame(self.current_frame_index).get_image(channel=self.channel)
self.image_shape = self.image.shape
self.create_windows()
self.set_target_absent()
self.image_buffer = {}
self._last_update = time.time()
self.refresh_timer = threading.Timer(0.05, self.refresh)
self.refresh_timer.start()
self.needs_refresh = True
self.trim_end = len(self.movie)-1
self.trim_start = 0
def fill_up_the_buffer(self):
r0 = max(self.current_frame_index-5, 0)
r1 = min(self.current_frame_index+5, len(self.movie))
for i in xrange(r0, r1, 1):
if i not in self.image_buffer.keys():
self.image_buffer[i] = self.movie.Frame(i).get_image(channel=self.channel)
for i in self.image_buffer.keys():
if i < r0 or i > r1:
del self.image_buffer[i]
def create_windows(self):
''' Create playback progress bar (allow user to move along the movie) '''
cv2.namedWindow(self.win_name)
cv2.moveWindow(self.win_name, 150, 150)
cv2.namedWindow("completeness")
cv2.moveWindow("completeness", 150, 50)
cv2.setMouseCallback(self.win_name, self.trackbar_window_onMouse)
cv2.createTrackbar("Frame", self.win_name, 0, len(self.movie) - 1, self.jump_to_frame_callback)
cv2.createTrackbar("Trim start", self.win_name, 0, len(self.movie) - 1, self.set_trim_start)
cv2.createTrackbar("Trim end", self.win_name, len(self.movie) - 1, len(self.movie) - 1, self.set_trim_end)
im_height = 30 # 30 pixels high completeness bar
im_width = len(self.movie)
self.completeness_image = np.zeros((im_height, im_width, 3))
current_frame_index = int(im_width * float(self.current_frame_index) / len(self.movie))
self.completeness_image[:, current_frame_index, :] = 1.
self.completeness = np.zeros(im_width, dtype=np.uint8)
for i in xrange(len(self.movie)):
bounds = self.movie.Frame(i).get_label(channel=self.channel, target=self.target)
if bounds is None or bounds.empty:
if bounds is None:
self.movie.Frame(i).set_label(channel=self.channel, target=self.target, label=BoundingRegion())
self.completeness[i] = 2
else:
if bounds.is_keyframe():
self.completeness[i] = 1
else:
self.completeness[i] = 0
def set_trim_start(self, frame_index):
self.trim_start = frame_index + 0
def set_trim_end(self, frame_index):
self.trim_end = frame_index + 0
def jump_to_frame_callback(self, frame_index):
self.current_frame_index = frame_index
self.image = self.movie.Frame(self.current_frame_index).get_image(channel=self.channel)
self.needs_refresh = True
def update_completeness_window(self):
if self.ready_to_refresh.acquire(False):
self.completeness_image *= 0
self.completeness_image[:, self.current_frame_index, :] = 255
for i in xrange(len(self.movie)):
self.completeness_image[:, i, self.completeness[i]] = 255
cv2.putText(self.completeness_image, '%i' % self.current_frame_index,
(10, 10), cv2.FONT_HERSHEY_PLAIN, 1.0, (255, 255, 255), lineType=cv2.CV_AA)
cv2.imshow('completeness', self.completeness_image)
self.ready_to_refresh.release()
def update_trackbar_window(self):
if self.ready_to_refresh.acquire():
image = self.image.copy()
self._stored_bounds = self.movie.Frame(self.current_frame_index).get_label(channel=self.channel, target=self.target)
# display rectangle over selected area
self._current_bounds.draw_box(image, color=(255, 255, 255))
self._current_bounds.draw_box(image, color=(0, 0, 0), thickness=1)
# display rectangle over stored area
if self._stored_bounds is not None:
self._stored_bounds.draw_box(image, color=(255, 0, 0))
if self._tracked_bounds is not None:
self._tracked_bounds.draw_box(image, color=(0, 255, 0))
# reshape to display if too small
min_width = 320
if image.shape[1] < min_width:
resize_height = int(0.5 + image.shape[0] * min_width / float(image.shape[1]))
image = cv2.resize(image, (min_width, resize_height), interpolation=cv2.INTER_NEAREST)
self.display_image=image
self.ready_to_refresh.release()
def refresh(self):
if self.timer_lock.acquire():
if self.needs_refresh:
self.update_trackbar_window()
self.update_completeness_window()
self.needs_refresh = False
self.refresh_timer = threading.Timer(0.05, self.refresh)
self.refresh_timer.start()
self.timer_lock.release()
def trackbar_window_onMouse(self, event, x, y, flags, _):
cv2.imshow(self.win_name, self.display_image)
# Update state
self.right_button_pressed = (flags & cv2.EVENT_FLAG_RBUTTON) != 0 and event != cv2.EVENT_RBUTTONUP
self.left_button_pressed = (flags & cv2.EVENT_FLAG_LBUTTON) != 0 and event != cv2.EVENT_LBUTTONUP
# Set bounding box
if event == cv2.EVENT_MBUTTONDOWN:
if self._anchor is None:
# Setting the anchor
self._anchor = (x, y)
else:
# Defining the box
self.save_and_advance()
elif self._anchor is not None and event != cv2.EVENT_MBUTTONUP:
# We are creating new bounding box from anchor to current position
self.x_size = max(5, int(abs(x - self._anchor[0])+0.5))
self.y_size = max(5, int(abs(y - self._anchor[1])+0.5))
center_x = (x + self._anchor[0]) / 2
center_y = (y + self._anchor[1]) / 2
self.set_bounding_box_with_center(center_x, center_y)
elif event == cv2.EVENT_MBUTTONUP:
self.x_size = max(5, int(abs(x - self._anchor[0])+0.5))
self.y_size = max(5, int(abs(y - self._anchor[1])+0.5))
center_x = (x + self._anchor[0]) / 2
center_y = (y + self._anchor[1]) / 2
self.set_bounding_box_with_center(center_x, center_y)
keyframe = False
if flags & cv2.EVENT_FLAG_CTRLKEY != 0:
keyframe = True
self.save_and_advance(keyframe=keyframe)
else:
self.set_bounding_box_with_center(x, y)
# If this is a left/right button down event,
# we can advance one frame immediately
# and then start the timer
if event == cv2.EVENT_RBUTTONDOWN:
self.set_target_absent()
self.save_and_advance()
self.right_button_pressed=True
if self.timer.finished:
# Rearm the timer
self.schedule_callback()
elif event == cv2.EVENT_LBUTTONDOWN:
self.set_bounding_box_with_center(x, y)
keyframe = False
if flags & cv2.EVENT_FLAG_CTRLKEY != 0:
keyframe = True
self.save_and_advance(keyframe=keyframe)
self.left_button_pressed=True
if self.timer.finished:
# Rearm the timer
self.schedule_callback()
elif not (self.right_button_pressed or self.left_button_pressed):
# No mouse button pressed, so cancel the timer if it's running
self.timer.cancel()
if time.time()-self._last_update > 0.1:
self._last_update = time.time()
self.needs_refresh = True
def set_target_absent(self):
self._current_bounds = BoundingRegion()
def set_bounding_box_with_center(self, x, y):
box = [np.clip(int(x+0.5)-self.x_size/2, 0, self.image_shape[1]),
np.clip(int(y+0.5)-self.y_size/2, 0, self.image_shape[0]),
self.x_size + min(int(x+0.5)-self.x_size/2, 0),
self.y_size + min(int(y+0.5)-self.y_size/2, 0)]
self._current_bounds = BoundingRegion(box=box)
self.needs_refresh = True
def schedule_callback(self):
# 0.2 secs => 5 Hz
self.timer = threading.Timer(0.1, self.timer_callback)
self.timer.start()
def timer_callback(self):
if self.right_button_pressed or self.left_button_pressed:
self.save_and_advance()
self.schedule_callback()
def advance_current_frame(self, increment=1):
self.current_frame_index = min(len(self.movie)-1, max(0, self.current_frame_index + increment))
self.image = self.movie.Frame(self.current_frame_index).get_image(channel=self.channel)
cv2.setTrackbarPos("Frame", self.win_name, self.current_frame_index)
if self.tracker is not None:
self._tracked_bounds = self.tracker.track(self.movie.Frame(self.current_frame_index).get_image(channel=self.channel))
def save_and_advance(self, keyframe=False):
self._anchor = None
self.movie.Frame(self.current_frame_index).set_label(self._current_bounds.copy(), channel=self.channel, target=self.target)
if self._current_bounds.empty:
self.completeness[self.current_frame_index] = 2
else:
if self._current_bounds.is_keyframe():
self.completeness[self.current_frame_index] = 1
else:
self.completeness[self.current_frame_index] = 0
if keyframe:
self.make_keyframe()
self.advance_current_frame(15)
else:
self.advance_current_frame(1)
def toggle_tracker(self):
if self.tracker is None:
self.tracker = CMTVisionTracker()
self.tracker.prime(self.movie.Frame(self.current_frame_index).get_image(channel=self.channel), self._current_bounds)
logging.warning("Tracker is enabled")
else:
self.tracker = None
self._tracked_bounds = None
logging.warning("Tracker is disabled")
def advance_tracker(self):
if self._tracked_bounds is not None:
self._current_bounds=self._tracked_bounds
self.save_and_advance()
def export_movie(self):
trim_frames_front = self.trim_start
trim_frames_end = (len(self.movie) -1) - self.trim_end
if trim_frames_end > 0:
for i in range(trim_frames_end):
self.movie.delete(-1)
if trim_frames_front>0:
for i in range(trim_frames_front):
self.movie.delete(0)
logging.warning("Exporting result in file " + self.output_file)
self.movie.write_to_file(self.output_file)
logging.warning("Exporting completed !")
def interpolate(self, start, end):
print "Interpolating %d %d" % (start, end)
label0 = self.movie.Frame(start).get_label(channel=self.channel, target=self.target)
label1 = self.movie.Frame(end).get_label(channel=self.channel, target=self.target)
if (label0.empty) or (label1.empty):
return
box0 = label0.get_box_pixels()
box1 = label1.get_box_pixels()
for i in xrange(start+1, end, 1):
alpha = (end-i)*1.0/(end-start)
box2 = alpha*np.array(box0)+(1-alpha)*np.array(box1)
box2 = map(lambda x: int(x), box2)
self.movie.Frame(i).set_label(BoundingRegion(box=box2, image_shape=self.movie.Frame(i).get_image(channel=self.channel).shape), channel=self.channel, target=self.target)
self.completeness[i] = 0
def make_keyframe(self):
self.completeness[self.current_frame_index] = 1
self.movie.Frame(self.current_frame_index).get_label(channel=self.channel, target=self.target).set_keyframe(True)
# find previous keyframe
next_keyframe = -1
for i in xrange(self.current_frame_index+1, len(self.movie), 1):
bounds = self.movie.Frame(i).get_label(channel=self.channel, target=self.target)
if bounds.is_keyframe():
next_keyframe = i
break
if next_keyframe >= 0:
print next_keyframe
self.interpolate(self.current_frame_index, next_keyframe)
# find next keyframe
prev_keyframe = -1
for i in xrange(self.current_frame_index-1, -1, -1):
bounds = self.movie.Frame(i).get_label(channel=self.channel, target=self.target)
if bounds.is_keyframe():
prev_keyframe = i
break
if prev_keyframe >= 0:
print prev_keyframe
self.interpolate(prev_keyframe, self.current_frame_index)
def run(self):
while True:
key = cv2.waitKey(0)
cv2.imshow(self.win_name, self.display_image)
if key == 65361: # left arrow
# Go to previous frame
self.advance_current_frame(-1)
elif key == 65362:
self.advance_current_frame(-10)
elif key == 65363: # right arrow
# Go to next frame
self.advance_current_frame()
elif key == 65364:
self.advance_current_frame(10)
elif key == ord('w'): # Write
self.export_movie()
if self.timer_lock.acquire():
if not self.refresh_timer.finished:
self.refresh_timer.cancel()
self.timer_lock.release()
self.timer.cancel()
self.reset()
elif key == ord('t'): # Tracker
self.toggle_tracker()
elif key == ord(' '): # Advance tracker
self.advance_tracker()
self.needs_refresh = True
elif key == ord('a'):
self.x_size = int(self.x_size*1.1)
self.y_size = int(self.y_size*1.1)
self.needs_refresh = True
elif key == ord('z'):
self.x_size = int(self.x_size*1/1.1)
self.y_size = int(self.y_size*1/1.1)
self.needs_refresh = True
elif key == ord('s'):
self.y_size = int(self.y_size*1.1)
self.needs_refresh = True
elif key == ord('k'):
self.make_keyframe()
elif key == ord('x'):
self.y_size = int(self.y_size*1/1.1)
self.needs_refresh = True
elif key == ord('q'): # Export
while key != -1:
for k in range(10):
key = cv2.waitKey(10)
print "Quiting! Do you want to save your work? [Y/N]"
while True:
key = cv2.waitKey(0)
if key == ord('Y') or key == ord('y'):
self.export_movie()
break
if key == ord('N') or key == ord('n'):
break
break
logging.warning('Exiting')
self.refresh_timer.cancel()