def handle_keyboard(self, key_mapper: KeyMapper):
par = self.par
if key_mapper.consume("n"):
par.context.mark_is_dashcam(not par.context.is_dashcam)
self.log("Marked video as {0}".format(
"dashcam" if par.context.is_dashcam else "not dashcam"))
elif key_mapper.consume("t"):
window = AdditionalTagWindow()
tags = window.get_user_tags()
par.context.set_additional_tags(tags)
self.log("Additional tags set")
elif key_mapper.consume("v"):
window = MultiSelectPopup("Select custom enum tags",
"video_enum_tags",
self.par.context.enum_tags)
enum_tags = window.run()
if enum_tags is not None:
self.log("Updated from {0}".format(self.par.context.enum_tags))
self.log("Now: {0}".format(enum_tags))
self.par.context.enum_tags = enum_tags
self.log("Remember to commit any new tags!")
else:
self.log("Enum tag update cancelled")
elif key_mapper.consume("l"):
ind = self.par.vcm.get_frame_index()
if self.par.bbm.has_accident_location(ind):
self.par.bbm.remove_accident_location(ind)
self.log("Accident location removed:{0}".format(ind))
else:
self.par.bbm.add_accident_location(ind)
self.log("Accident location added:{0}".format(ind))
self.log("Is now {0}".format(
self.par.bbm.get_accident_locations()))
def handle_keyboard(self, key_mapper: KeyMapper):
par = self.par
if key_mapper.consume("n"):
par.context.mark_is_dashcam(not par.context.is_dashcam)
self.log("Marked video as {0}".format(
"dashcam" if par.context.is_dashcam else "not dashcam"))
elif key_mapper.consume("t"):
window = AdditionalTagWindow()
tags = window.get_user_tags()
par.context.set_additional_tags(tags)
self.log("Additional tags set")
def handle_keyboard(self, key_mapper: KeyMapper):
par = self.par
if key_mapper.consume("n"):
par.context.mark_is_dashcam(not par.context.is_dashcam)
self.log("Marked video as {0}".format(
"dashcam" if par.context.is_dashcam else "not dashcam"))
elif key_mapper.consume("t"):
window = AdditionalTagWindow()
tags = window.get_user_tags()
par.context.set_additional_tags(tags)
self.log("Additional tags set")
elif key_mapper.consume("v"):
window = MultiSelectPopup("Select custom enum tags",
"video_enum_tags",
self.par.context.enum_tags)
enum_tags = window.run()
if enum_tags is not None:
self.log("Updated from {0}".format(self.par.context.enum_tags))
self.log("Now: {0}".format(enum_tags))
self.par.context.enum_tags = enum_tags
self.log("Remember to commit any new tags!")
else:
self.log("Enum tag update cancelled")
elif key_mapper.consume("l"):
ind = self.par.vcm.get_frame_index()
if self.par.bbm.has_collision_location(ind):
self.par.bbm.remove_collision_location(ind)
self.log("collision location removed:{0}".format(ind))
else:
self.par.bbm.add_collision_location(ind)
self.log("collision location added:{0}".format(ind))
self.log("Is now {0}".format(
self.par.bbm.get_collision_locations()))
elif key_mapper.consume("i"):
ind = self.par.vcm.get_frame_index()
if self.par.bbm.has_reckless_start_frame():
self.par.bbm.set_reckless_end_frame(ind)
self.log("Reckless interval created:({0}, {1})"\
.format(self.par.bbm.get_reckless_start_frame(), ind))
self.par.bbm.clear_reckless_start()
else:
self.par.bbm.set_reckless_start_frame(ind)
self.log("Reckless start frame set:{0}".format(ind))
elif key_mapper.consume("u"):
ind = self.par.vcm.get_frame_index()
removed = self.par.bbm.clear_reckless_frames(ind)
self.log("Remove reckless frames:{0}".format(removed))
elif key_mapper.consume("k"):
par.context.mark_to_be_deleted(not par.context.to_be_deleted)
self.log("Marked video to {0}".format(
"be deleted" if par.context.to_be_deleted else "not be deleted"
))
elif key_mapper.consume("o"):
new_location = SelectPopup("Enter location of accident",
"video_locations", 10,
VIDEO_LOCATION_DEFAULT_OPTIONS).run()
if new_location is not None:
par.context.set_location(new_location)
self.log(f"Updated Location to {new_location}")
def __init__(self, context: VideoTaggingContext):
self.context = context
self.vcm = self.context.vcm
self.frame_rate = 25
self.logger = RotatingLog(self.LOG_LINES)
self.ignore_index_change_interval = self.vcm.get_frames_count() // 200
self.bbm = BoundingBoxManager(self.vcm.get_frames_count())
self.bbm.set_to(*context.get_bbox_fields_as_list())
self.mode_handlers = [
InternaSelectionMode(self),
InternalBBoxMode(self)
]
self.mode_handler_i = 0
self.instructions = GENERAL_INSTRUCTIONS
self.key_mapper = KeyMapper()
def __init__(self, context: BBContext, mode_handlers: list):
self.context = context
self.vcm = self.context.vcm
self.frame_rate = 25
self.logger = RotatingLog(self.LOG_LINES)
self.ignore_index_change_interval = self.vcm.get_frames_count() // 200
self.mode_handlers = mode_handlers
self.mode_handler_i = 0
self.key_mapper = KeyMapper()
self.bbm = BoundingBoxManager(self.vcm.get_frames_count())
self.bbm.set_to(
context,
selected=[
obj.id for obj in context.bbox_fields.objects.values()
if obj.has_collision
],
)
def handle_keyboard(self, key_mapper: KeyMapper):
par = self.par
bbm = par.bbm
if key_mapper.consume("i"): # Select id
id = TextPopup(
"Enter ID. If ID exists, will work on original").run()
if id is None or id == "":
self.cancel("Select ID",
"Still on {0}".format(self.selected_id))
else:
try:
id = int(id)
self.log("Switched id from {0} to {1}".format(
self.selected_id, id))
self.selected_id = id
self.reset_task()
except:
self.error("Input ID not valid. Still on {0}".format(
self.selected_id))
elif key_mapper.consume("u"): # Overwrite class
if not bbm.has_id(self.selected_id):
self.error(
"Could not update class. ID {0} does not exist".format(
self.selected_id))
else:
cls = SelectPopup("Enter new class for the selected object",
"bb_classes", 10,
BB_CLASS_DEFAULT_OPTIONS).run()
prev = bbm.get_cls(self.selected_id)
if cls == None:
self.cancel(
"Class change", "{0} still on class {1}".format(
self.selected_id, prev))
else:
bbm.add_or_update_id(self.selected_id, cls)
self.log("Class for ID {0} updated from {1} to {2}".format(
self.selected_id, prev, cls))
elif key_mapper.consume("r"):
self.reset_task()
self.log("Selected bounding boxes reset".format(self.selected_id))
elif key_mapper.consume("p"):
deleted_ids = bbm.remove_unused_ids()
self.log("Removed {0} ids without bounding boxes: {1}".format(
len(deleted_ids), list(deleted_ids)))
elif key_mapper.consume("z"):
removed = self.im.remove_last()
if removed is None:
self.error("No input to remove")
else:
self.log("Removed last bounding box input on frame {0}".format(
removed.i))
elif key_mapper.consume("x"):
removed = self.im.remove_last()
if removed is None:
self.error("No input to change")
else:
prev_i = removed.i
removed.i = self.par.vcm.get_frame_index()
self.im.add(removed)
self.log(
"Moved last bounding box input from frame {0} to frame {1}"
.format(prev_i, removed.i))
elif key_mapper.consume("q"):
index = self.par.vcm.get_frame_index()
retrived = bbm.get_last_bounding_box_i(self.selected_id, index)
if retrived is None:
self.error(
"Retrieving previous frame with bounding box failed")
else:
self.im.add(bbm.get_ir(self.selected_id, retrived))
elif key_mapper.consume("e"):
index = self.par.vcm.get_frame_index()
retrived = bbm.get_next_bounding_box_i(self.selected_id, index)
if retrived is None:
self.error("Retrieving next frame with bounding box failed")
else:
self.im.add(bbm.get_ir(self.selected_id, retrived))
elif key_mapper.consume("b"):
if self.im.has_n(2):
if not bbm.has_id(self.selected_id):
bbm.add_or_update_id(self.selected_id, self.DEFAULT_CLASS)
self.log("New id {0} for class {1} added".format(
self.selected_id, self.DEFAULT_CLASS))
bbm.replace_in_range(self.selected_id, *self.im.get_2_sorted())
self.log(
"Bounding box for ID {0} set over index range [{1}, {2}]".
format(self.selected_id, self.im[0].i, self.im[1].i))
elif self.im.has_n(1):
if not bbm.has_id(self.selected_id):
bbm.add_or_update_id(self.selected_id, self.DEFAULT_CLASS)
self.log("New id {0} for class {1} added".format(
self.selected_id, self.DEFAULT_CLASS))
bbm.replace_in_range(self.selected_id, self.im.get_last(),
self.im.get_last())
self.log("Single bounding box for ID {0} set at {1}".format(
self.selected_id,
self.im.get_last().i))
else:
self.error(
"Not enough inputs. Command ignored. Please draw at least 1 bounding box"
)
elif key_mapper.consume("c"):
if not self.im.has_n(2):
self.error(
"Not enough inputs. Command ignored. Please click on 2 frames"
)
elif not bbm.has_id(self.selected_id):
self.error(
"Cannot clear bboxes - ID {0} does not exist".format(
self.selected_id))
else:
i1 = self.im[0].i
i2 = self.im[1].i
bbm.clear_in_range(self.selected_id, i1, i2)
self.log("Bbox for ID {0} cleared over index range [{1}, {2}]".
format(self.selected_id, i1, i2))
unused_ids = bbm.get_unused_ids()
if len(unused_ids) > 0:
self.warn("Exists ids without bounding box: {0}".format(
list(unused_ids)))
self.hint("Press p to remove")
elif key_mapper.consume("v"):
if not bbm.has_id(self.selected_id):
self.error(
"Cannot clear bboxes - ID {0} does not exist".format(
self.selected_id))
else:
i = self.par.vcm.get_frame_index()
bbm.clear_in_range(self.selected_id, i, i)
self.log("Bbox for ID {0} cleared for a single frame".format(
self.selected_id, i))
unused_ids = bbm.get_unused_ids()
if len(unused_ids) > 0:
self.warn("Exists ids without bounding box: {0}".format(
list(unused_ids)))
self.hint("Press p to remove")
class VideoPlayerGUIManager(object):
PROGRESS_BAR_NAME = "progress"
FRAME_RATE_BAR_NAME = "frame_delay"
PAUSE_BUTTON_NAME = "pause"
WINDOW_NAME = 'overwriteThis'
LOG_LINES = 6
LOG_LINE_HEIGHT = 17
LOG_LINE_MARGIN = 2
LOG_START_X = 250
IMG_STARTING_Y = LOG_LINE_HEIGHT * LOG_LINES + LOG_LINE_MARGIN * (
LOG_LINES + 1) + 3
INSTRUCTIONS = [
["m or tab", "Switch mode"],
["h", "Open instructions page"],
["a", "1 back"],
["s", "10 back"],
["d", "1 forward"],
["w", "10 forward"],
["Space", "Pause/unpause"],
["Enter * 2", "Finish and continue"],
[
"Esc * 2",
"Abort and restart tagging. Will raise ManualTaggingAbortedException"
],
]
def __init__(self, context: BBContext, mode_handlers: list):
self.context = context
self.vcm = self.context.vcm
self.frame_rate = 25
self.logger = RotatingLog(self.LOG_LINES)
self.ignore_index_change_interval = self.vcm.get_frames_count() // 200
self.mode_handlers = mode_handlers
self.mode_handler_i = 0
self.key_mapper = KeyMapper()
self.bbm = BoundingBoxManager(self.vcm.get_frames_count())
self.bbm.set_to(
context,
selected=[
obj.id for obj in context.bbox_fields.objects.values()
if obj.has_collision
],
)
def start(self):
self.set_GUI()
try:
self.play_video()
except ManualTaggingAbortedException:
raise
finally:
self.cleanup()
def set_GUI(self):
cv2.namedWindow(self.WINDOW_NAME)
cv2.resizeWindow(self.WINDOW_NAME, self.vcm.get_width(),
self.vcm.get_height() + self.IMG_STARTING_Y)
cv2.setMouseCallback(
self.WINDOW_NAME,
lambda event, x, y, flags, param: self.handleClick(
event, x, y, flags, param))
def set_frame_rate_callback(value):
self.frame_rate = max(1, value)
def set_progress_rate_callback(value):
if abs(value - self.vcm.get_frame_index()) > self.ignore_index_change_interval or \
value == 0 or value == self.vcm.get_frames_count()-1:
self.vcm.start_from(value)
def set_paused_callback(value):
if self.vcm is not None:
self.vcm.set_paused(value)
cv2.createTrackbar(self.PROGRESS_BAR_NAME, self.WINDOW_NAME, 0,
max(0,
self.vcm.get_frames_count() - 1),
set_progress_rate_callback)
cv2.createTrackbar(self.FRAME_RATE_BAR_NAME, self.WINDOW_NAME,
self.frame_rate, 200, set_frame_rate_callback)
cv2.createTrackbar(self.PAUSE_BUTTON_NAME, self.WINDOW_NAME, False, 1,
set_paused_callback)
def play_video(self):
shown_for_first_time = False
while True:
if shown_for_first_time and cv2.getWindowProperty(
self.WINDOW_NAME,
cv2.WND_PROP_VISIBLE) <= 0: # Window closed. Abort
raise ManualTaggingExitedException(
"Tagging operation aborted by closing window")
frame = self.vcm.next().copy()
frame_index = self.vcm.get_frame_index()
frame = self.modify_frame(frame, frame_index)
frame = self.get_mode_handler().modify_frame(frame, frame_index)
cv2.imshow(self.WINDOW_NAME, self.build_frame(frame))
shown_for_first_time = True
cv2.setTrackbarPos(self.PROGRESS_BAR_NAME, self.WINDOW_NAME,
frame_index)
self.key_mapper.append(cv2.waitKey(self.frame_rate) & 0xFF)
if self.key_mapper.consume(("esc", "esc")): # Escape key
res = ButtonPopup(
"Confirm restart",
"Hitting confirm will destroy all progress. You will have to restart. Continue?",
["Confirm", "Cancel"],
).run()
if res == "Confirm":
raise ManualTaggingAbortedException(
"Tagging operation aborted")
elif self.key_mapper.consume(("enter", "enter")): # Enter
if not self.can_commit():
self.logger.log("[ERROR] Commit operation failed")
else:
res = ButtonPopup("Confirm commit",
self.get_commit_message(),
["Confirm", "Cancel"]).run()
if res == "Confirm":
break
elif self.key_mapper.consume("h"):
window = HelpPopup(
"GUI controls reference",
self.INSTRUCTIONS + [["", ""]] +
self.get_mode_handler().INSTRUCTIONS,
)
window.run()
elif self.key_mapper.consume("a"):
self.vcm.shift_frame_index(-1)
elif self.key_mapper.consume("s"):
self.vcm.shift_frame_index(-10)
elif self.key_mapper.consume("d"):
self.vcm.shift_frame_index(1)
elif self.key_mapper.consume("w"):
self.vcm.shift_frame_index(10)
elif self.key_mapper.consume(" "):
cv2.setTrackbarPos(self.PAUSE_BUTTON_NAME, self.WINDOW_NAME,
0 if self.vcm.get_paused() else 1)
if self.key_mapper.consume("m") or self.key_mapper.consume("tab"):
self.mode_handler_i += 1
self.mode_handler_i %= len(self.mode_handlers)
self.logger.log("Changed mode")
else:
self.get_mode_handler().handle_keyboard(self.key_mapper)
def build_frame(self, frame):
img = np.zeros(
(self.context.file_height + self.IMG_STARTING_Y,
self.context.file_width, 3),
np.uint8,
)
def write_top_text():
font = cv2.FONT_HERSHEY_SIMPLEX
font_scale = 0.5
font_color = (255, 255, 255)
for i, msg in enumerate(self.logger.get_logs()):
starting_index = (self.LOG_START_X, self.LOG_LINE_HEIGHT *
(i + 1) + self.LOG_LINE_MARGIN * i)
cv2.putText(img, msg, starting_index, font, font_scale,
font_color)
for i, msg in enumerate(
self.get_mode_handler().get_state_message()):
starting_index = (0, self.LOG_LINE_HEIGHT * (i + 1) +
self.LOG_LINE_MARGIN * i)
cv2.putText(img, msg, starting_index, font, font_scale,
font_color)
write_top_text()
displayed = cv2.cvtColor(frame, cv2.IMREAD_COLOR)
img[self.IMG_STARTING_Y:, 0:] = displayed
return img
def handleClick(self, event, x, y, flags, param):
y = y - self.IMG_STARTING_Y
self.get_mode_handler().handle_click(event, x, y, flags, param)
def get_mode_handler(self):
return self.mode_handlers[self.mode_handler_i]
def cleanup(self):
self.vcm.release()
cv2.destroyAllWindows()
def modify_frame(self, frame, frame_index):
raise NotImplementedError()
def can_commit(self):
raise NotImplementedError()
def get_commit_message(self):
return "Hitting confirm will commit all changes. You will not be able to undo any changes afterwards. Continue?"
class VideoPlayerGUIManager(object):
PROGRESS_BAR_NAME = "progress"
FRAME_RATE_BAR_NAME = "frame_delay"
PAUSE_BUTTON_NAME = "pause"
WINDOW_NAME = 'tagger'
LOG_LINES = 6
LOG_LINE_HEIGHT = 17
LOG_LINE_MARGIN = 2
LOG_START_X = 250
IMG_STARTING_Y = LOG_LINE_HEIGHT * LOG_LINES + LOG_LINE_MARGIN * (
LOG_LINES + 1) + 3
def __init__(self, context: VideoTaggingContext):
self.context = context
self.vcm = self.context.vcm
self.frame_rate = 25
self.logger = RotatingLog(self.LOG_LINES)
self.ignore_index_change_interval = self.vcm.get_frames_count() // 200
self.bbm = BoundingBoxManager(self.vcm.get_frames_count())
self.bbm.set_to(*context.get_bbox_fields_as_list())
self.mode_handlers = [
InternaSelectionMode(self),
InternalBBoxMode(self)
]
self.mode_handler_i = 0
self.instructions = GENERAL_INSTRUCTIONS
self.key_mapper = KeyMapper()
def start(self):
self.set_GUI()
try:
self.logger.log("Starting with: {0} bounding box ids".format(
self.bbm.get_n_ids()))
self.play_video()
except ManualTaggingAbortedException:
raise
finally:
self.cleanup()
self.context.set_bbox_fields_from_list(self.bbm.extract())
def set_GUI(self):
cv2.namedWindow(self.WINDOW_NAME)
cv2.setMouseCallback(
self.WINDOW_NAME,
lambda event, x, y, flags, param: self.handleClick(
event, x, y, flags, param))
def set_frame_rate_callback(value):
self.frame_rate = max(1, value)
def set_progress_rate_callback(value):
if abs(value - self.vcm.get_frame_index()) > self.ignore_index_change_interval or \
value == 0 or value == self.vcm.get_frames_count()-1:
self.vcm.start_from(value)
def set_paused_callback(value):
if self.vcm is not None:
self.vcm.set_paused(value)
cv2.createTrackbar(self.PROGRESS_BAR_NAME, self.WINDOW_NAME, 0,
max(0,
self.vcm.get_frames_count() - 1),
set_progress_rate_callback)
cv2.createTrackbar(self.FRAME_RATE_BAR_NAME, self.WINDOW_NAME,
self.frame_rate, 200, set_frame_rate_callback)
cv2.createTrackbar(self.PAUSE_BUTTON_NAME, self.WINDOW_NAME, False, 1,
set_paused_callback)
def play_video(self):
shown_for_first_time = False
while True:
if shown_for_first_time and cv2.getWindowProperty(
self.WINDOW_NAME,
cv2.WND_PROP_VISIBLE) <= 0: # Window closed. Abort
raise ManualTaggingExitedException(
"Tagging operation aborted by closing window")
frame = self.vcm.next().copy()
frame_index = self.vcm.get_frame_index()
frame = self.bbm.modify_frame(frame, frame_index)
frame = self.get_mode_handler().modify_frame(frame, frame_index)
cv2.imshow(self.WINDOW_NAME, self.build_frame(frame))
shown_for_first_time = True
cv2.setTrackbarPos(self.PROGRESS_BAR_NAME, self.WINDOW_NAME,
frame_index)
self.key_mapper.append(cv2.waitKey(self.frame_rate) & 0xFF)
if self.key_mapper.consume("esc"): # Escape key
res = ButtonPopup(
"Confirm restart",
"Hitting confirm will destroy all progress. You will have to restart. Continue?",
["Confirm", "Cancel"]).run()
if res == "Confirm":
raise ManualTaggingAbortedException(
"Tagging operation aborted")
elif self.key_mapper.consume("enter"): # Enter
res = ButtonPopup(
"Confirm commit",
"Hitting confirm will commit all changes. You will not be able to undo any changes afterwards. Continue?",
["Confirm", "Cancel"]).run()
if res == "Confirm":
break
elif self.key_mapper.consume("h"):
window = HelpPopup(
"GUI controls reference", self.instructions + [["", ""]] +
self.get_mode_handler().instructions)
window.run()
elif self.key_mapper.consume("a"):
self.vcm.shift_frame_index(-1)
elif self.key_mapper.consume("s"):
self.vcm.shift_frame_index(-10)
elif self.key_mapper.consume("d"):
self.vcm.shift_frame_index(1)
elif self.key_mapper.consume("w"):
self.vcm.shift_frame_index(10)
elif self.key_mapper.consume(" "):
cv2.setTrackbarPos(self.PAUSE_BUTTON_NAME, self.WINDOW_NAME,
0 if self.vcm.get_paused() else 1)
if self.key_mapper.consume("m"):
self.mode_handler_i += 1
self.mode_handler_i %= len(self.mode_handlers)
self.logger.log("Changed mode")
else:
self.get_mode_handler().handle_keyboard(self.key_mapper)
def build_frame(self, frame):
img = np.zeros((self.context.file_height + self.IMG_STARTING_Y,
self.context.file_width, 3), np.uint8)
def write_top_text():
font = cv2.FONT_HERSHEY_DUPLEX
font_scale = 0.5
font_color = (255, 255, 255)
for i, msg in enumerate(self.logger.get_logs()):
starting_index = (self.LOG_START_X, self.LOG_LINE_HEIGHT *
(i + 1) + self.LOG_LINE_MARGIN * i)
cv2.putText(img,
msg,
starting_index,
font,
font_scale,
font_color,
lineType=cv2.LINE_AA)
for i, msg in enumerate(
self.get_mode_handler().get_state_message()):
starting_index = (0, self.LOG_LINE_HEIGHT * (i + 1) +
self.LOG_LINE_MARGIN * i)
cv2.putText(img,
msg,
starting_index,
font,
font_scale,
font_color,
lineType=cv2.LINE_AA)
write_top_text()
displayed = cv2.cvtColor(frame, cv2.IMREAD_COLOR)
img[self.IMG_STARTING_Y:, 0:] = displayed
return img
def handleClick(self, event, x, y, flags, param):
y = y - self.IMG_STARTING_Y
self.get_mode_handler().handle_click(event, x, y, flags, param)
def get_mode_handler(self):
return self.mode_handlers[self.mode_handler_i]
def cleanup(self):
self.vcm.release()
cv2.destroyAllWindows()