def __str__(self):
content = "ChangeRegion -> ID: " + str(self.number_id) + "\n"
content += " Time: " + TimeHelper.stampToStr(self.creation_time)
content += " - " + TimeHelper.stampToStr(self.last_modified)
if self.locked_time >= 0.0:
content += " - " + TimeHelper.stampToStr(self.locked_time)
if self.erased_time is None:
content += "\n"
else:
content += " - " + TimeHelper.stampToStr(self.erased_time) + "\n"
content += " Region: X: [" + str(self.min_x) + ", " + str(
self.max_x) + "]"
content += ", Y: [" + str(self.min_y) + ", " + str(self.max_y) + "]\n"
content += " Cells: " + str(len(self.cells))
return content
def update_selected_keyframe(self, new_selected):
if 0 <= new_selected < len(self.keyframe_annotations):
self.selected_keyframe = new_selected
else:
return
self.lbl_nav_keyframe.set_text("Key-Frame: " + str(self.selected_keyframe + 1) + " / " +
str(len(self.keyframe_annotations)))
time_str = TimeHelper.stampToStr(self.keyframe_annotations[self.selected_keyframe].time)
self.lbl_nav_time.set_text(time_str)
self.update_current_view()
def start_input_processing(self, process_function):
for lecture in self.database.lectures:
self.current_lecture = lecture
m_videos, lecture_file, skip = self.get_lecture_params(lecture)
if skip:
continue
# read temporal file
if self.input_temp_prefix is None:
# null-input process (convenient way to process lectures)
input_data = None
else:
if not isinstance(self.input_temp_prefix, list):
input_data = MiscHelper.dump_load(self.temp_dir + '/' +
self.input_temp_prefix +
lecture_file)
else:
input_data = []
for temp_prefix in self.input_temp_prefix:
input_data.append(
MiscHelper.dump_load(self.temp_dir + '/' +
temp_prefix + lecture_file))
# execute the actual process ....
timer = TimeHelper()
timer.startTimer()
results = process_function(self, input_data)
timer.endTimer()
print("Process Finished in: " + timer.totalElapsedStamp())
# save results
if self.output_temp_prefix is not None:
if not isinstance(self.output_temp_prefix, list):
MiscHelper.dump_save(
results, self.temp_dir + '/' +
self.output_temp_prefix + lecture_file)
else:
for out_idx, temp_prefix in enumerate(
self.output_temp_prefix):
MiscHelper.dump_save(
results[out_idx],
self.temp_dir + '/' + temp_prefix + lecture_file)
def doProcessing(self, video_worker, limit=0, verbose=False):
# initially....
width = None
height = None
offset_frame = -1
absolute_frame = 0
absolute_time = 0.0
last_frame = None
next_sample_frame_idx = 0
if verbose:
print("Video processing for " + video_worker.getWorkName() +
" has begun")
# for timer...
timer = TimeHelper()
timer.startTimer()
# open video...
for video_idx, video_file in enumerate(self.file_list):
try:
capture = cv2.VideoCapture(video_file)
except Exception as e:
# error loading
raise Exception("The file <" + video_file +
"> could not be opened")
capture_width = int(capture.get(cv2.CAP_PROP_FRAME_WIDTH))
capture_height = int(capture.get(cv2.CAP_PROP_FRAME_HEIGHT))
# ...check size ...
forced_resizing = False
if width is None:
# first video....
# initialize local parameters....
if self.forced_width is not None:
# ...size...
width = self.forced_width
height = self.forced_height
if capture_width != self.forced_width or capture_height != self.forced_height:
forced_resizing = True
else:
width = capture_width
height = capture_height
# on the worker class...
video_worker.initialize(width, height)
else:
if self.forced_width is not None:
forced_resizing = (capture_width != self.forced_width
or capture_height != self.forced_height)
else:
if (width != capture_width) or (height != capture_height):
# invalid, all main video files must be the same resolution...
raise Exception(
"All video files on the list must have the same resolution"
)
# Read video until the end or until limit has been reached
while (limit == 0 or offset_frame < limit
) and not next_sample_frame_idx >= len(self.frame_list):
if offset_frame == self.frame_list[next_sample_frame_idx]:
# grab and decode next frame since it is in the list
flag, frame = capture.read()
else:
# just grab to move forward in the list
flag = capture.grab()
# print("Grab time: " + str(capture.get(cv2.CAP_PROP_POS_FRAMES)))
# print((valid_grab, flag, type(frame), selection_step, jump_frames))
if not flag:
# end of video reached...
break
if offset_frame == self.frame_list[next_sample_frame_idx]:
# continue ..
current_time = capture.get(cv2.CAP_PROP_POS_MSEC)
current_frame = capture.get(cv2.CAP_PROP_POS_FRAMES)
if forced_resizing:
frame = cv2.resize(
frame, (self.forced_width, self.forced_height))
frame_time = absolute_time + current_time
frame_idx = int(absolute_frame + current_frame)
video_worker.handleFrame(frame, last_frame, video_idx,
frame_time, current_time,
frame_idx)
if verbose:
print("Frames Processed = {0:d}, Video Time = {1:s}".
format(offset_frame,
TimeHelper.stampToStr(frame_time)))
last_frame = frame
next_sample_frame_idx += 1
if next_sample_frame_idx >= len(self.frame_list):
# end of sample reached ...
break
offset_frame += 1
# at the end of the processing of current video
capture.set(cv2.CAP_PROP_POS_AVI_RATIO, 1.0)
video_length = capture.get(cv2.CAP_PROP_POS_MSEC)
video_frames = capture.get(cv2.CAP_PROP_POS_FRAMES)
absolute_time += video_length
absolute_frame += video_frames
# processing finished...
video_worker.finalize()
# end time counter...
timer.endTimer()
if verbose:
print("Video processing for " + video_worker.getWorkName() +
" completed: " +
TimeHelper.stampToStr(timer.lastElapsedTime() * 1000.0))
def __init__(self, size, db_name, lecture_title, output_path):
Screen.__init__(self, "Formula Ground Truth Annotation Interface",
size)
general_background = (50, 80, 160)
text_color = (255, 255, 255)
button_text_color = (20, 20, 50)
button_back_color = (228, 228, 228)
self.elements.back_color = general_background
self.db_name = db_name
self.lecture_title = lecture_title
self.output_path = output_path
export_filename = self.output_path + "/segments.xml"
export_image_prefix = self.output_path + "/keyframes/"
# load including segment information
self.keyframe_annotations, self.segments = KeyFrameAnnotation.LoadExportedKeyframes(
export_filename, export_image_prefix, True)
if len(self.keyframe_annotations) > 0:
print("Key-frames Loaded: " + str(len(self.keyframe_annotations)))
else:
raise Exception("Cannot start with 0 key-frames")
portions_filename = self.output_path + "/portions.xml"
portions_path = self.output_path + "/portions/"
if os.path.exists(portions_filename):
# Saved data detected, loading
print("Previously saved portion data detected, loading")
KeyFrameAnnotation.LoadKeyframesPortions(portions_filename,
self.keyframe_annotations,
portions_path)
else:
raise Exception("No saved portion data detected, cannot continue")
print("Original Key-frames: " + str(len(self.keyframe_annotations)))
print("Segments: " + str(len(self.segments)))
self.keyframe_annotations = KeyFrameAnnotation.CombineKeyframesPerSegment(
self.keyframe_annotations, self.segments, True)
print("Key-frames after combination per segment" +
str(len(self.keyframe_annotations)))
# other CC/group elements
self.unique_groups = None
self.cc_group = None
self.cc_total = 0
for kf_idx, keyframe in enumerate(self.keyframe_annotations):
self.cc_total += len(keyframe.binary_cc)
unique_cc_filename = self.output_path + "/unique_ccs.xml"
if os.path.exists(unique_cc_filename):
# Saved data detected, loading
print("Previously saved unique CC data detected, loading")
self.cc_group, self.unique_groups = UniqueCCGroup.GroupsFromXML(
self.keyframe_annotations, unique_cc_filename)
else:
# no previous data, build default index (all CCs are unique)
raise Exception(
"No unique CC data found for lecture. Must label Unique CC first"
)
self.view_mode = GTFormulaAnnotator.ViewModeColored
self.edition_mode = GTFormulaAnnotator.ModeNavigate
self.view_scale = 1.0
self.selected_keyframe = 0
self.selected_formula = 0
self.adding_groups = []
self.formulas_per_frame = [
[] for idx in range(len(self.keyframe_annotations))
]
saved_filename = self.output_path + "/formula_ccs.xml"
if os.path.exists(saved_filename):
# load saved data ...
self.formulas_ccs = FormulaCCs.FormulasFromXML(
self.unique_groups, saved_filename)
# add to index per frame ...
for formula in self.formulas_ccs:
for frame_idx in range(formula.first_visible,
formula.last_visible + 1):
self.formulas_per_frame[frame_idx].append(formula)
print("Loaded: " + saved_filename)
else:
self.formulas_ccs = []
# update draw cache of highlighted formulae ...
self.colored_cache = [None] * len(self.keyframe_annotations)
for idx in range(len(self.keyframe_annotations)):
self.update_colored_cache(idx)
# add elements....
container_top = 10
container_width = 330
button_2_width = 150
button_2_left = int(container_width * 0.25) - button_2_width / 2
button_2_right = int(container_width * 0.75) - button_2_width / 2
# Navigation panel to move accross frames
self.container_nav_buttons = ScreenContainer(
"container_nav_buttons", (container_width, 70),
back_color=general_background)
self.container_nav_buttons.position = (
self.width - self.container_nav_buttons.width - 10, container_top)
self.elements.append(self.container_nav_buttons)
self.lbl_nav_keyframe = ScreenLabel(
"lbl_nav_keyframe",
"Key-Frame: 1 / " + str(len(self.keyframe_annotations)), 21, 290,
1)
self.lbl_nav_keyframe.position = (5, 5)
self.lbl_nav_keyframe.set_background(general_background)
self.lbl_nav_keyframe.set_color(text_color)
self.container_nav_buttons.append(self.lbl_nav_keyframe)
time_str = TimeHelper.stampToStr(
self.keyframe_annotations[self.selected_keyframe].time)
self.lbl_nav_time = ScreenLabel("lbl_nav_time", time_str, 21, 290, 1)
self.lbl_nav_time.position = (5,
self.lbl_nav_keyframe.get_bottom() + 20)
self.lbl_nav_time.set_background(general_background)
self.lbl_nav_time.set_color(text_color)
self.container_nav_buttons.append(self.lbl_nav_time)
self.btn_nav_keyframe_prev = ScreenButton("btn_nav_keyframe_prev",
"Prev", 21, 90)
self.btn_nav_keyframe_prev.set_colors(button_text_color,
button_back_color)
self.btn_nav_keyframe_prev.position = (
10, self.lbl_nav_keyframe.get_bottom() + 10)
self.btn_nav_keyframe_prev.click_callback = self.btn_nav_keyframe_prev_click
self.container_nav_buttons.append(self.btn_nav_keyframe_prev)
self.btn_nav_keyframe_next = ScreenButton("btn_nav_keyframe_next",
"Next", 21, 90)
self.btn_nav_keyframe_next.set_colors(button_text_color,
button_back_color)
self.btn_nav_keyframe_next.position = (
self.container_nav_buttons.width -
self.btn_nav_keyframe_next.width - 10,
self.lbl_nav_keyframe.get_bottom() + 10)
self.btn_nav_keyframe_next.click_callback = self.btn_nav_keyframe_next_click
self.container_nav_buttons.append(self.btn_nav_keyframe_next)
# ================================================================================
# confirmation panel
self.container_confirm_buttons = ScreenContainer(
"container_confirm_buttons", (container_width, 70),
back_color=general_background)
self.container_confirm_buttons.position = (
self.width - self.container_confirm_buttons.width - 10,
container_top)
self.elements.append(self.container_confirm_buttons)
self.container_confirm_buttons.visible = False
self.lbl_confirm_message = ScreenLabel(
"lbl_confirm_message", "Confirmation message goes here?", 21, 290,
1)
self.lbl_confirm_message.position = (5, 5)
self.lbl_confirm_message.set_background(general_background)
self.lbl_confirm_message.set_color(text_color)
self.container_confirm_buttons.append(self.lbl_confirm_message)
self.btn_confirm_cancel = ScreenButton("btn_confirm_cancel", "Cancel",
21, 130)
self.btn_confirm_cancel.set_colors(button_text_color,
button_back_color)
self.btn_confirm_cancel.position = (
10, self.lbl_nav_keyframe.get_bottom() + 10)
self.btn_confirm_cancel.click_callback = self.btn_confirm_cancel_click
self.container_confirm_buttons.append(self.btn_confirm_cancel)
self.btn_confirm_accept = ScreenButton("btn_confirm_accept", "Accept",
21, 130)
self.btn_confirm_accept.set_colors(button_text_color,
button_back_color)
self.btn_confirm_accept.position = (
self.container_confirm_buttons.width -
self.btn_confirm_accept.width - 10,
self.lbl_confirm_message.get_bottom() + 10)
self.btn_confirm_accept.click_callback = self.btn_confirm_accept_click
self.container_confirm_buttons.append(self.btn_confirm_accept)
# ================================================================================
# View panel with view control buttons
self.container_view_buttons = ScreenContainer(
"container_view_buttons", (container_width, 165),
back_color=general_background)
self.container_view_buttons.position = (
self.width - self.container_view_buttons.width - 10,
self.container_nav_buttons.get_bottom() + 10)
self.elements.append(self.container_view_buttons)
button_width = 190
button_left = (self.container_view_buttons.width - button_width) / 2
# zoom ....
self.lbl_zoom = ScreenLabel("lbl_zoom", "Zoom: 100%", 21,
container_width - 10, 1)
self.lbl_zoom.position = (5, 5)
self.lbl_zoom.set_background(general_background)
self.lbl_zoom.set_color(text_color)
self.container_view_buttons.append(self.lbl_zoom)
self.btn_zoom_reduce = ScreenButton("btn_zoom_reduce", "[ - ]", 21, 90)
self.btn_zoom_reduce.set_colors(button_text_color, button_back_color)
self.btn_zoom_reduce.position = (10, self.lbl_zoom.get_bottom() + 10)
self.btn_zoom_reduce.click_callback = self.btn_zoom_reduce_click
self.container_view_buttons.append(self.btn_zoom_reduce)
self.btn_zoom_increase = ScreenButton("btn_zoom_increase", "[ + ]", 21,
90)
self.btn_zoom_increase.set_colors(button_text_color, button_back_color)
self.btn_zoom_increase.position = (self.container_view_buttons.width -
self.btn_zoom_increase.width - 10,
self.lbl_zoom.get_bottom() + 10)
self.btn_zoom_increase.click_callback = self.btn_zoom_increase_click
self.container_view_buttons.append(self.btn_zoom_increase)
self.btn_zoom_zero = ScreenButton("btn_zoom_zero", "100%", 21, 90)
self.btn_zoom_zero.set_colors(button_text_color, button_back_color)
self.btn_zoom_zero.position = (
(self.container_view_buttons.width - self.btn_zoom_zero.width) / 2,
self.lbl_zoom.get_bottom() + 10)
self.btn_zoom_zero.click_callback = self.btn_zoom_zero_click
self.container_view_buttons.append(self.btn_zoom_zero)
self.btn_view_raw = ScreenButton("btn_view_raw", "Raw View", 21,
button_2_width)
self.btn_view_raw.set_colors(button_text_color, button_back_color)
self.btn_view_raw.position = (button_2_left,
self.btn_zoom_zero.get_bottom() + 10)
self.btn_view_raw.click_callback = self.btn_view_raw_click
self.container_view_buttons.append(self.btn_view_raw)
self.btn_view_gray = ScreenButton("btn_view_gray", "Grayscale View",
21, button_2_width)
self.btn_view_gray.set_colors(button_text_color, button_back_color)
self.btn_view_gray.position = (button_2_right,
self.btn_zoom_zero.get_bottom() + 10)
self.btn_view_gray.click_callback = self.btn_view_gray_click
self.container_view_buttons.append(self.btn_view_gray)
self.btn_view_binary = ScreenButton("btn_view_binary", "Binary View",
21, button_2_width)
self.btn_view_binary.set_colors(button_text_color, button_back_color)
self.btn_view_binary.position = (button_2_left,
self.btn_view_gray.get_bottom() + 10)
self.btn_view_binary.click_callback = self.btn_view_binary_click
self.container_view_buttons.append(self.btn_view_binary)
self.btn_view_colored = ScreenButton("btn_view_colored",
"Colored View", 21,
button_2_width)
self.btn_view_colored.set_colors(button_text_color, button_back_color)
self.btn_view_colored.position = (button_2_right,
self.btn_view_gray.get_bottom() + 10)
self.btn_view_colored.click_callback = self.btn_view_colored_click
self.container_view_buttons.append(self.btn_view_colored)
# =================================================================================
# Panel with action buttons (Add/Remove links)
self.container_action_buttons = ScreenContainer(
"container_action_buttons", (container_width, 240),
general_background)
self.container_action_buttons.position = (
self.container_view_buttons.get_left(),
self.container_view_buttons.get_bottom() + 5)
self.elements.append(self.container_action_buttons)
self.lbl_nav_formula = ScreenLabel("lbl_nav_formula", "X / X", 21,
container_width - 10, 1)
self.lbl_nav_formula.position = (5, 5)
self.lbl_nav_formula.set_background(general_background)
self.lbl_nav_formula.set_color(text_color)
self.container_action_buttons.append(self.lbl_nav_formula)
self.btn_nav_formula_prev = ScreenButton("btn_nav_formula_prev",
"Prev", 21, button_2_width)
self.btn_nav_formula_prev.set_colors(button_text_color,
button_back_color)
self.btn_nav_formula_prev.position = (
button_2_left, self.lbl_nav_formula.get_bottom() + 10)
self.btn_nav_formula_prev.click_callback = self.btn_nav_formula_prev_click
self.container_action_buttons.append(self.btn_nav_formula_prev)
self.btn_nav_formula_next = ScreenButton("btn_nav_formula_next",
"Next", 21, button_2_width)
self.btn_nav_formula_next.set_colors(button_text_color,
button_back_color)
self.btn_nav_formula_next.position = (
button_2_right, self.lbl_nav_formula.get_bottom() + 10)
self.btn_nav_formula_next.click_callback = self.btn_nav_formula_next_click
self.container_action_buttons.append(self.btn_nav_formula_next)
self.btn_formulas_add = ScreenButton("btn_formulas_add", "Add Formula",
21, button_2_width)
self.btn_formulas_add.set_colors(button_text_color, button_back_color)
self.btn_formulas_add.position = (
button_2_left, self.btn_nav_formula_next.get_bottom() + 20)
self.btn_formulas_add.click_callback = self.btn_formulas_add_click
self.container_action_buttons.append(self.btn_formulas_add)
self.btn_formulas_del = ScreenButton("btn_formulas_del",
"Del. Formula", 21,
button_2_width)
self.btn_formulas_del.set_colors(button_text_color, button_back_color)
self.btn_formulas_del.position = (
button_2_right, self.btn_nav_formula_next.get_bottom() + 20)
self.btn_formulas_del.click_callback = self.btn_formulas_del_click
self.container_action_buttons.append(self.btn_formulas_del)
self.btn_formula_update_tag = ScreenButton("btn_formula_update_tag",
"Update Tag", 21,
button_width)
self.btn_formula_update_tag.set_colors(button_text_color,
button_back_color)
self.btn_formula_update_tag.position = (
button_left, self.btn_formulas_del.get_bottom() + 20)
self.btn_formula_update_tag.click_callback = self.btn_formula_update_tag_click
self.container_action_buttons.append(self.btn_formula_update_tag)
self.lbl_formula_tag = ScreenLabel("lbl_formula_tag", "Tag: ?", 21,
container_width - 10, 1)
self.lbl_formula_tag.position = (
5, self.btn_formula_update_tag.get_bottom() + 20)
self.lbl_formula_tag.set_background(general_background)
self.lbl_formula_tag.set_color(text_color)
self.container_action_buttons.append(self.lbl_formula_tag)
# ===============================================
stats_background = (60, 50, 40)
self.container_stats = ScreenContainer("container_stats",
(container_width, 70),
back_color=stats_background)
self.container_stats.position = (
self.width - container_width - 10,
self.container_action_buttons.get_bottom() + 5)
self.elements.append(self.container_stats)
self.lbl_cc_stats = ScreenLabel("lbl_cc_stats",
"Connected Component Stats", 21,
container_width - 10, 1)
self.lbl_cc_stats.position = (5, 5)
self.lbl_cc_stats.set_background(stats_background)
self.lbl_cc_stats.set_color(text_color)
self.container_stats.append(self.lbl_cc_stats)
self.lbl_cc_raw = ScreenLabel("lbl_cc_raw",
"Total Raw CC:\n" + str(self.cc_total),
21, button_2_width, 1)
self.lbl_cc_raw.position = (button_2_left,
self.lbl_cc_stats.get_bottom() + 10)
self.lbl_cc_raw.set_background(stats_background)
self.lbl_cc_raw.set_color(text_color)
self.container_stats.append(self.lbl_cc_raw)
self.lbl_cc_unique = ScreenLabel(
"lbl_cc_unique",
"Total Unique CC:\n" + str(len(self.unique_groups)), 21,
button_2_width, 1)
self.lbl_cc_unique.position = (button_2_right,
self.lbl_cc_stats.get_bottom() + 10)
self.lbl_cc_unique.set_background(stats_background)
self.lbl_cc_unique.set_color(text_color)
self.container_stats.append(self.lbl_cc_unique)
#=============================================================
# Panel with state buttons (Undo, Redo, Save)
self.container_state_buttons = ScreenContainer(
"container_state_buttons", (container_width, 200),
general_background)
self.container_state_buttons.position = (
self.container_view_buttons.get_left(),
self.container_stats.get_bottom() + 10)
self.elements.append(self.container_state_buttons)
self.btn_undo = ScreenButton("btn_undo", "Undo", 21, button_width)
self.btn_undo.set_colors(button_text_color, button_back_color)
self.btn_undo.position = (button_left, 5)
self.btn_undo.click_callback = self.btn_undo_click
self.container_state_buttons.append(self.btn_undo)
self.btn_redo = ScreenButton("btn_redo", "Redo", 21, button_width)
self.btn_redo.set_colors(button_text_color, button_back_color)
self.btn_redo.position = (button_left, self.btn_undo.get_bottom() + 10)
self.btn_redo.click_callback = self.btn_redo_click
self.container_state_buttons.append(self.btn_redo)
self.btn_save = ScreenButton("btn_save", "Save", 21, button_width)
self.btn_save.set_colors(button_text_color, button_back_color)
self.btn_save.position = (button_left, self.btn_redo.get_bottom() + 10)
self.btn_save.click_callback = self.btn_save_click
self.container_state_buttons.append(self.btn_save)
self.btn_exit = ScreenButton("btn_exit", "Exit", 21, button_width)
self.btn_exit.set_colors(button_text_color, button_back_color)
self.btn_exit.position = (button_left, self.btn_save.get_bottom() + 30)
self.btn_exit.click_callback = self.btn_exit_click
self.container_state_buttons.append(self.btn_exit)
# ==============================================================
image_width = self.width - self.container_nav_buttons.width - 30
image_height = self.height - container_top - 10
self.container_images = ScreenContainer("container_images",
(image_width, image_height),
back_color=(0, 0, 0))
self.container_images.position = (10, container_top)
self.elements.append(self.container_images)
# ... image objects ...
tempo_blank = np.zeros((50, 50, 3), np.uint8)
tempo_blank[:, :, :] = 255
self.img_main = ScreenImage("img_main", tempo_blank, 0, 0, True,
cv2.INTER_NEAREST)
self.img_main.position = (0, 0)
self.img_main.mouse_button_down_callback = self.img_mouse_down
self.container_images.append(self.img_main)
# canvas used for annotations
self.canvas_select = ScreenCanvas("canvas_select", 100, 100)
self.canvas_select.position = (0, 0)
self.canvas_select.locked = False
# self.canvas_select.object_edited_callback = self.canvas_object_edited
# self.canvas_select.object_selected_callback = self.canvas_selection_changed
self.container_images.append(self.canvas_select)
self.canvas_select.add_element("selection_rectangle", 10, 10, 40, 40)
self.canvas_select.elements["selection_rectangle"].visible = False
self.undo_stack = []
self.redo_stack = []
self.update_selected_formula(0)
self.update_current_view(True)
def computeVisualAlignment(m_videos, a_videos, time_offset, motionless,
save_frames, extraction_method_id):
#distribute the selection of motionless frames...
selected = MiscHelper.distribute_values(Aligner.ALIGNMENT_SAMPLE, 0,
len(motionless) - 1)
#create the list..
frame_list = []
for idx in selected:
frame_list.append(motionless[idx])
#extract the motionless frames from main videos
frames = Loader.extractFramesRelative(m_videos, frame_list)
if save_frames:
for idx, f in enumerate(frames):
abs_time, frame = f
cv2.imwrite("out/main_" + str(idx) + ".jpg", frame)
#calculate the absolute time for the corresponding frames
#on the auxiliar video. Consider the time difference between videos
times = [(abs_time - time_offset) for abs_time, frame in frames]
#extract the motionless frames from auxiliar videos
aux_frames = Loader.extractFramesAbsolute(a_videos, times)
if save_frames:
for idx, frame in enumerate(aux_frames):
cv2.imwrite("out/auxiliar_" + str(idx) + ".jpg", frame)
#find the visual correspondence between pairs of key frames
matches_aux = []
matches_main = []
aux_boxes = []
main_boxes = []
all_content_main = []
all_content_aux = []
#...first... extract the content from each pair of frames...
for i in range(min(Aligner.ALIGNMENT_SAMPLE, len(frames))):
#get the current key frames
abs_time, frame_main = frames[i]
frame_aux = aux_frames[i]
print("Extracting content #" + str(i + 1) + " ... (Main: " +
TimeHelper.stampToStr(abs_time) + " - Aux: " +
TimeHelper.stampToStr(times[i]) + ")")
#from the main key frame, extract content on the board
main_box, content_main = Binarizer.frameContentBinarization(
frame_main, extraction_method_id)
main_boxes.append(main_box)
#from the auxiliary key frame, extract content on the board
aux_box, content_aux = Binarizer.frameContentBinarization(
frame_aux, extraction_method_id)
aux_boxes.append(aux_box)
#add to list...
all_content_main.append(content_main)
all_content_aux.append(content_aux)
#...then, extract the alignment.... keep highest score...
all_scores = []
for i in range(min(Aligner.ALIGNMENT_SAMPLE, len(frames))):
print("Testing Alignment #" + str(i + 1) + " ... ")
#corresponding frames....
content_aux = all_content_aux[i]
content_main = all_content_main[i]
#Extract a set of good matches between these two images....
# where object = aux content from mimio, to align with main content
# scene = main content to which the change regions will be projected
aux_list, main_list = VisualAlignment.getSURFMatchingPoints(
content_aux, content_main, Aligner.SURF_THRESHOLD)
#generate projection based on these points...
current_projection, mask = VisualAlignment.generateProjection(
aux_list, main_list)
#calculate score...
score = VisualAlignment.getProjectionScore(current_projection,
all_content_main,
all_content_aux)
#print( str(i) + " => " + str(score) )
all_scores.append((score, i, current_projection))
#add to the total list of points...
matches_aux.append(aux_list)
matches_main.append(main_list)
#print( "ON " + str(i) + " where found " + str(len(aux_list) ) + " matches" )
all_scores = sorted(all_scores, reverse=True)
#current best projection is the one with the top score...
max_score = all_scores[0][0]
all_matches_aux = matches_aux[all_scores[0][1]]
all_matches_main = matches_main[all_scores[0][1]]
best_projection = all_scores[0][2]
#now, try to improve the quality of the projection by adding some keypoints from
#candidate alignments with high scores and computing a new combined projection
#for the list of combined keypoint matches...
new_score = max_score
pos = 1
while new_score >= max_score and pos < len(all_scores):
#add keypoints to the combined list...
current_aux = all_matches_aux + matches_aux[all_scores[pos][1]]
current_main = all_matches_main + matches_main[all_scores[pos][1]]
#generate the new projection...
current_projection, mask = VisualAlignment.generateProjection(
current_aux, current_main)
#get score for combined projection...
new_score = VisualAlignment.getProjectionScore(
current_projection, all_content_main, all_content_aux)
#check if score improved...
if new_score >= max_score:
#new best projection found....
max_score = new_score
all_matches_aux += aux_list[all_scores[pos][1]]
all_matches_main += main_list[all_scores[pos][1]]
best_projection = current_projection
pos += 1
#Get the final alignment
projection = best_projection
print("Best Alignment Score: " + str(max_score))
"""
# Un-comment to output alignment images
for i in range(len(all_content_main)):
content_main = all_content_main[i]
content_aux = all_content_aux[i]
proj_img = np.zeros( (content_main.shape[0], content_main.shape[1]), dtype=content_main.dtype )
cv.WarpPerspective( cv.fromarray( content_aux ), cv.fromarray(proj_img), cv.fromarray( projection ) )
result_image = np.zeros( (content_main.shape[0], content_main.shape[1], 3) )
result_image[:,:,2] = content_main
result_image[:,:,1] = proj_img
#cv2.imshow('img',result_image)
cv2.imwrite( 'DEBUG_MAIN_' + str(i) + '.bmp', content_main )
cv2.imwrite( 'DEBUG_AUX_' + str(i) + '.bmp', content_aux )
cv2.imwrite( 'DEBUG_PROJECTION_' + str(i) + '.bmp' , result_image )
"""
#average of the boxes of the whiteboard
main_box = MiscHelper.averageBoxes(main_boxes)
aux_box = MiscHelper.averageBoxes(aux_boxes)
#store them in a single object...
visual_alignment = VisualAlignment()
# ... main size...
visual_alignment.main_width = frames[0][1].shape[1]
visual_alignment.main_height = frames[0][1].shape[0]
#.... main board box ...
visual_alignment.main_box = main_box
# ... aux size ....
visual_alignment.aux_width = aux_frames[0].shape[1]
visual_alignment.aux_height = aux_frames[0].shape[0]
#... aux board box...
visual_alignment.aux_box = aux_box
#... projection ....
visual_alignment.projection = projection
return visual_alignment
def __repr__(self):
lect_str = self.database + " - " + self.lecture
loc_str = str(self.idx) + " at " + TimeHelper.stampToStr(self.time)
return "{Keyframe: [" + lect_str + "], [" + loc_str + "]}\n"
def doProcessing(self,
video_worker,
limit=0,
verbose=False,
force_no_seek=False):
#initially....
width = None
height = None
offset_frame = -1
absolute_frame = 0
absolute_time = 0.0
if verbose:
print("Video processing for " + video_worker.getWorkName() +
" has begun")
#for timer...
timer = TimeHelper()
timer.startTimer()
#open video...
for video_idx, video_file in enumerate(self.file_list):
try:
capture = cv2.VideoCapture(video_file)
except Exception as e:
# error loading
raise Exception("The file <" + video_file +
"> could not be opened")
capture_width = int(capture.get(cv2.CAP_PROP_FRAME_WIDTH))
capture_height = int(capture.get(cv2.CAP_PROP_FRAME_HEIGHT))
# ...check size ...
forced_resizing = False
if width is None:
# first video....
# initialize local parameters....
if self.forced_width is not None:
# ...size...
width = self.forced_width
height = self.forced_height
if capture_width != self.forced_width or capture_height != self.forced_height:
forced_resizing = True
else:
width = capture_width
height = capture_height
# on the worker class...
video_worker.initialize(width, height)
else:
if self.forced_width is not None:
forced_resizing = (capture_width != self.forced_width
or capture_height != self.forced_height)
else:
if (width != capture_width) or (height != capture_height):
# invalid, all main video files must be the same resolution...
raise Exception(
"All video files on the list must have the same resolution"
)
# get current FPS
video_fps = capture.get(cv2.CAP_PROP_FPS)
# will use some frames per second
jump_frames = int(video_fps / self.frames_per_second)
# Read video until the end or until limit has been reached
selection_step = 4 if force_no_seek else 1
timer_1 = TimeHelper()
timer_2 = TimeHelper()
while limit == 0 or offset_frame < limit:
if selection_step == 2 or selection_step == 5:
# jump to frame in single step
timer_2.startTimer()
target_frame = capture.get(
cv2.CAP_PROP_POS_FRAMES) + jump_frames - 1
valid_grab = capture.set(cv2.CAP_PROP_POS_FRAMES,
target_frame)
timer_2.endTimer()
if selection_step == 2:
selection_step = 3
if selection_step == 1 or selection_step == 4:
timer_1.startTimer()
# jump to frame by grabbing frames ...
valid_grab = True
for x in range(jump_frames - 1):
valid_grab = capture.grab()
if not valid_grab:
break
timer_1.endTimer()
if selection_step == 1:
selection_step = 2
if selection_step == 3:
# decide which sampling grabbing method is faster
if timer_1.totalElapsedTime() < timer_2.totalElapsedTime():
print("Grabbing frames to jump")
selection_step = 4
else:
print("Jumping to frames directly")
selection_step = 5
# get frame..
if valid_grab:
flag, frame = capture.read()
else:
flag, frame = False, None
#print("Grab time: " + str(capture.get(cv2.CAP_PROP_POS_FRAMES)))
#print((valid_grab, flag, type(frame), selection_step, jump_frames))
if not flag:
# end of video reached...
break
else:
offset_frame += 1
current_time = capture.get(cv2.CAP_PROP_POS_MSEC)
current_frame = capture.get(cv2.CAP_PROP_POS_FRAMES)
if forced_resizing:
frame = cv2.resize(frame,
(self.forced_width, self.forced_height))
if offset_frame > 0:
frame_time = absolute_time + current_time
frame_idx = int(absolute_frame + current_frame)
video_worker.handleFrame(frame, last_frame, video_idx,
frame_time, current_time,
frame_idx)
if verbose and offset_frame % 50 == 0:
print( "Frames Processed = " + str(offset_frame) + \
", Video Time = " + TimeHelper.stampToStr( frame_time ) )
last_frame = frame
last_time = current_time
#at the end of the processing of current video
capture.set(cv2.CAP_PROP_POS_AVI_RATIO, 1.0)
video_length = capture.get(cv2.CAP_PROP_POS_MSEC)
video_frames = capture.get(cv2.CAP_PROP_POS_FRAMES)
absolute_time += video_length
absolute_frame += video_frames
#processing finished...
video_worker.finalize()
#end time counter...
timer.endTimer()
if verbose:
print("Video processing for " + video_worker.getWorkName() +
" completed: " +
TimeHelper.stampToStr(timer.lastElapsedTime() * 1000.0))
def extractRegions(main_videos, speaker_detector, change_detector,
alignment, time_offset):
#sort the input regions
#by last time modified...
regions = change_detector.getAllRegions()
sorted_regions = [(r.last_modified, idx, r)
for idx, r in enumerate(regions)]
sorted_regions = sorted(sorted_regions)
open_capture = None
open_index = None
extracted_parts = {}
extracted_boxes = {}
w_cells = change_detector.getWidthCells()
h_cells = change_detector.getHeightCells()
motions = speaker_detector.getMotionDetected()
#now extract them from the videos
for time_modified, idx, r in sorted_regions:
#for the region r, check one frame between
#last time modified and time locked where
#it is assumed to be not obstructed...
real_time_modified = time_modified + time_offset
init_index = speaker_detector.findMotionByTime(real_time_modified)
if init_index == len(motions):
init_index -= 1
candidates = []
#find the corresponding region of interest in original video...
#...first... from cells to rectangle in original frame....
cx_min = (r.min_x / float(w_cells)) * alignment.aux_width
cx_max = ((r.max_x + 1) / float(w_cells)) * alignment.aux_width
cy_min = (r.min_y / float(h_cells)) * alignment.aux_height
cy_max = ((r.max_y + 1) / float(h_cells)) * alignment.aux_height
#...then... use scaling information to obtain corresponding
#... rectangle in main video....
main_region = alignment.alignRegion(cx_min, cx_max, cy_min, cy_max)
#<THE MASK>
"""
#generate the mask
mask = r.getMask(h_cells, w_cells)
#resize...
new_size = (int(mask.shape[1] * ChangeDetector.CELL_SIZE), int(mask.shape[0] * ChangeDetector.CELL_SIZE) )
mask = cv2.resize(mask, new_size)
#project the mask
proj_mask = np.zeros( (alignment.main_height, alignment.main_width) , dtype='uint8' )
cv.WarpPerspective( cv.fromarray( mask ), cv.fromarray(proj_mask), cv.fromarray( alignment.projection ) )
#dilate the mask
#....create structuring element...
expand_cells = 4
strel = cv2.getStructuringElement(cv2.MORPH_RECT, (int(ChangeDetector.CELL_SIZE * expand_cells), int(ChangeDetector.CELL_SIZE * expand_cells)))
#....now dilate mask...
final_mask = cv2.dilate(proj_mask, strel)
"""
#</THE MASK>
#now, find a frame where the found region has no motion around
intervals = speaker_detector.getNonblockedIntervals(
main_region, alignment.main_width, alignment.main_height,
init_index, r.locked_time + time_offset)
#check....
if len(intervals) == 0:
#cannot be extracted on lock time...
#now do a second attempt to extract it based on erasing time...
if r.lock_type == ChangeRegion.COMPLETE_ON_OVERWRITE and \
r.erased_time != None:
intervals = speaker_detector.getNonblockedIntervals(
main_region, alignment.main_width,
alignment.main_height, init_index,
r.erased_time + time_offset)
#check....
if len(intervals) == 0:
#it simply could not be extracted....
extracted_parts[r.number_id] = None
extracted_boxes[r.number_id] = None
print( "CHECK = " + str(r.number_id ) + ", from " + \
TimeHelper.stampToStr(r.creation_time + time_offset) + " - " + \
TimeHelper.stampToStr(motions[init_index].absolute_time) + " to " + \
TimeHelper.stampToStr(r.locked_time + time_offset) )
else:
#find the best interval....
best_interval = 0
best_length = (intervals[0][1] - intervals[0][0] + 1)
for i in range(1, len(intervals)):
length = (intervals[i][1] - intervals[i][0] + 1)
if length > best_length:
best_length = length
best_interval = i
#now, from best interval pick frame in the middle..
best_frame = int(init_index +
((intervals[best_interval][0] +
intervals[best_interval][1]) / 2.0))
#finally do extraction
#... open video ...
if open_index == None or open_index != motions[
best_frame].video_index:
#close current video
if open_index != None:
open_capture.release()
open_index = motions[best_frame].video_index
open_capture = cv2.VideoCapture(main_videos[open_index])
#... set time position ...
open_capture.set(cv.CV_CAP_PROP_POS_MSEC,
motions[best_frame].time)
#... get frame ...
flag, frame = open_capture.read()
if not flag:
#error?
extracted_parts[r.number_id] = None
extracted_boxes[r.number_id] = None
print("Region <" + str(r.number_id) +
"> Could not be extracted from video")
else:
#extract the region ...
margin = 5
min_x = int(max(0, math.floor(main_region[0] - margin)))
max_x = int(
min(alignment.main_width - 1,
math.ceil(main_region[1] + margin)))
min_y = int(max(0, math.floor(main_region[2] - margin)))
max_y = int(
min(alignment.main_height - 1,
math.ceil(main_region[3] + margin)))
#get the part of the image...
part = frame[min_y:max_y, min_x:max_x, :]
#<THE MASK>
"""
part_mask = final_mask[min_y:max_y, min_x:max_x]
size_mask = int((part_mask.shape[0] * part_mask.shape[1]) - np.count_nonzero(part_mask))
if size_mask > 0:
original_part = part.copy()
blured_part = cv2.GaussianBlur(original_part, (41, 41), 4.0)
part[part_mask == 0, :] = blured_part[part_mask == 0, :]
"""
#</THE MASK>
extracted_parts[r.number_id] = part
extracted_boxes[r.number_id] = (min_x, max_x, min_y, max_y)
print("Region <" + str(r.number_id) +
"> extracted succesfully!")
return extracted_parts, extracted_boxes
def doProcessing(self, video_worker, limit=0, verbose=False):
# initially....
width = None
height = None
offset_frame = -1
absolute_frame = 0
absolute_time = 0.0
if verbose:
print("Video processing for " + video_worker.getWorkName() +
" has begun")
# for timer...
timer = TimeHelper()
timer.startTimer()
# open video...
try:
print(self.src_dir)
capture = ImageListGenerator(
'{}/{}'.format(self.src_dir, 'JPEGImages'), self.img_extension)
except Exception as e:
# error loading
print(e)
raise Exception(
"The directory <" + self.src_dir +
"> is not in the correct export format, check index.json")
last_frame = None
capture_width = capture.width
capture_height = capture.height
# print(capture_width, capture_height)
# ...check size ...
forced_resizing = False
if width is None:
# first video....
# initialize local parameters....
if self.forced_width is not None:
# ...size...
width = self.forced_width
height = self.forced_height
if capture_width != self.forced_width or capture_height != self.forced_height:
forced_resizing = True
else:
width = capture_width
height = capture_height
# on the worker class...
video_worker.initialize(width, height)
else:
if self.forced_width is not None:
forced_resizing = (capture_width != self.forced_width
or capture_height != self.forced_height)
else:
if (width != capture_width) or (height != capture_height):
# invalid, all main video files must be the same resolution...
raise Exception(
"All files on the list must have the same resolution")
while limit == 0 or offset_frame < limit:
# get frame..
flag, frame = capture.read()
if not flag:
# end of video reached...
print('end of video reached...')
break
else:
offset_frame += 1
current_time = capture.get('abs_time')
current_frame = capture.index2frameID()
if forced_resizing:
frame = cv2.resize(frame,
(self.forced_width, self.forced_height))
if offset_frame >= 0:
frame_time = absolute_time + current_time
frame_idx = int(absolute_frame + current_frame)
video_worker.handleFrame(frame, last_frame, 0, frame_time,
current_time, frame_idx)
if verbose and offset_frame % 50 == 0:
print("Frames Processed = " + str(offset_frame) +
", Video Time = " +
TimeHelper.stampToStr(frame_time))
last_frame = frame
last_time = current_time
# processing finished...
video_worker.finalize()
# end time counter...
timer.endTimer()
if verbose:
print("Video processing for " + video_worker.getWorkName() +
" completed: " +
TimeHelper.stampToStr(timer.lastElapsedTime() * 1000.0))
def main():
if len(sys.argv) < 2:
print("Usage:")
print("\tpython train_ml_binarizer.py config [force_update] [classifier_file] [patch_size]")
print("")
print("Where")
print("\tconfig\t\t\tPath to config file")
print("\tforce_update \t\tOptional, force to update the sampled Patch file")
print("\tclassifier_file \tOptional, force classifier path diff. from Config")
print("\tpatch_size \t\tOptional, override patch size")
return
# read the configuration file ....
config = Configuration.from_file(sys.argv[1])
# load the database
try:
database = MetaDataDB.from_file(config.get_str("VIDEO_DATABASE_PATH"))
except:
print("Invalid database file")
return
# <Parameters>
run_crossvalidation = config.get("ML_BINARIZER_TRAIN_RUN_CROSSVALIDATION", True)
if not config.contains("ML_BINARIZER_PATCHES_FILENAME"):
print("Must specificy a file to store sampled patches")
return
output_dir = config.get_str("OUTPUT_PATH")
ml_binarizer_dir = output_dir + "/" + config.get_str("ML_BINARIZER_DIR")
patch_filename = ml_binarizer_dir + "/" + config.get_str("ML_BINARIZER_PATCHES_FILENAME")
# For debugging/comparsion, use OTSU binarization
OTSU_mode = config.get("ML_BINARIZER_TRAIN_OTSU_MODE", False)
# baseline_mode = True # Train Random Forest instead .
retrain_classifier = config.get("ML_BINARIZER_TRAIN_RETRAIN", True)
if not config.get("ML_BINARIZER_OVERRIDE_PARAMETERS", False):
# Sampling mode #1: Distribution of proportions
#
# of 100% pixels, we sample fg_proportion from GT Foreground pixels (handwriting pixels)
# handwriting pixels = fg_proportion
# all background = (1 - fg_proportion)
#
# The remaining background pixels are sampled as close or far from foreground
# Close to Foreground pixels = (1 - fg_proportion) * bg_close_prop
# Remaining background pixels = (1 - fg_proportion) * (1 - bg_close_prop)
#
# The last proportion of pixels can be obtained from whiteboard or background objects, we separate them as
# Not Close Whiteboard background pixels = (1 - fg_proportion) * (1 - bg_close_prop) * bg_board_prop
# Not Whiteboard background pixels = (1 - fg_proportion) * (1 - bg_close_prop) * (1 - bg_board_prop)
#
# Sampling mode #2: Distribution of proportions
#
# of 100% pixels, we sample fg_proportion from GT Foreground pixels (handwriting pixels)
# handwriting pixels = fg_proportion
# all background = (1 - fg_proportion)
#
# The remaining background pixels by average intensity of the window, the greater the average, the more likely they
# are to be sampled. This is consistent with sampling mode 1, but is less discrete and requires less parameters
sampling_mode = Parameters.MLBin_sampling_mode
patch_size = Parameters.MLBin_patch_size
patches_per_frame = Parameters.MLBin_sampling_patches_per_frame
fg_proportion = Parameters.MLBin_sampling_fg_proportion
bg_close_prop = Parameters.MLBin_sampling_bg_close_prop
bg_board_prop = Parameters.MLBin_sampling_bg_board_prop
mlbin_sigma_color = Parameters.MLBin_sigma_color
mlbin_sigma_space = Parameters.MLBin_sigma_space
mlbin_median_blur_k = Parameters.MLBin_median_blur_k
mlbin_dark_background = Parameters.MLBin_dark_background
feature_workers = Parameters.MLBin_train_workers
# Random Forest
rf_n_trees = Parameters.MLBin_rf_n_trees # 16
rf_max_depth = Parameters.MLBin_rf_max_depth # 12
rf_max_features = Parameters.MLBin_rf_max_features # 32
else:
print("Reading ML Binarizer parameters from config ...")
sampling_mode = config.get_int("ML_BINARIZER_SAMPLING_MODE", 2)
patch_size = config.get_int("ML_BINARIZER_PATCH_SIZE", 7)
patches_per_frame = config.get_int("ML_BINARIZER_SAMPLING_PATCHES_PER_FRAME", 20000)
fg_proportion = config.get_float("ML_BINARIZER_SAMPLING_FG_PROPORTION", 0.5)
bg_close_prop = config.get_float("ML_BINARIZER_SAMPLING_BG_CLOSE_PROPORTION", 0.9)
bg_board_prop = config.get_float("ML_BINARIZER_SAMPLING_BG_BOARD_PROPORTION", 1.0)
mlbin_sigma_color = config.get_float("ML_BINARIZER_SIGMA_COLOR", 13.5)
mlbin_sigma_space = config.get_float("ML_BINARIZER_SIGMA_SPACE", 4.0)
mlbin_median_blur_k = config.get_int("ML_BINARIZER_MEDIAN_BLUR_K", 33)
mlbin_dark_background = config.get("ML_BINARIZER_DARK_BACKGROUND")
feature_workers = config.get_int("ML_BINARIZER_TRAIN_WORKERS", 7)
# Random Forest
rf_n_trees = config.get_int("ML_BINARIZER_RF_N_TREES", 16) # 16
rf_max_depth = config.get_int("ML_BINARIZER_RF_MAX_DEPTH", 12) # 12
rf_max_features = config.get_int("ML_BINARIZER_RF_MAX_FEATURES", 32) # 32
if len(sys.argv) >= 4:
# user specified location
classifier_file = sys.argv[3]
else:
# by default, store at the place specified in the configuration or parameters file ...
if not config.get("ML_BINARIZER_OVERRIDE_PARAMETERS", False):
classifier_file = Parameters.MLBin_classifier_file
else:
classifier_file = ml_binarizer_dir + "/" + config.get_str("ML_BINARIZER_CLASSIFIER_FILENAME")
feature_function = get_patch_features_raw_values
# </Parameters>
if len(sys.argv) >= 3:
try:
force_update = int(sys.argv[2]) > 0
except:
print("Invalid value for force_udpate")
return
else:
force_update = False
if len(sys.argv) >= 5:
try:
patch_size = int(sys.argv[4])
except:
print("Invalid value for patch_size")
return
assert (patch_size - 1) % 2 == 0
bg_close_neighborhood = int((patch_size - 1) / 2) + 1
print("Classifier Path: " + classifier_file)
ml_binarizer = MLBinarizer(None, patch_size, mlbin_sigma_color, mlbin_sigma_space, mlbin_median_blur_k,
mlbin_dark_background)
print("... loading data ...")
start_loading = time.time()
all_keyframes, binarized_keyframes = load_keyframes(output_dir, database)
fake_unique_groups, fake_cc_group, fake_segments = generate_fake_keyframe_info(all_keyframes)
print("Total Training keyframes: " + str(len(all_keyframes)))
end_loading = time.time()
start_preprocessing = time.time()
print("Pre-processing key-frames", flush=True)
all_preprocessed = []
for kf_idx, kf in enumerate(all_keyframes):
all_preprocessed.append(ml_binarizer.preprocessing(kf.raw_image))
# cv2.imwrite("DELETE_NOW_tempo_bin_input_" + str(kf_idx) + ".png", all_preprocessed[-1])
end_preprocessing = time.time()
start_patch_extraction = time.time()
# Extracting/Loading patches used for training (only if not on OTSU's mode)
if not OTSU_mode:
# generate the patch-based training set ...
# check if patch file exists ...
if not os.path.exists(patch_filename) or force_update:
print("Extracting patches...")
if sampling_mode == 1:
# SampleEdgeFixBg()
patches = PatchSampling.SampleEdgeFixBg(all_keyframes, all_preprocessed, patch_size, patches_per_frame,
fg_proportion, bg_close_prop, bg_board_prop, bg_close_neighborhood)
elif sampling_mode == 2:
# SampleEdgeContBg
patches = PatchSampling.SampleEdgeContBg(all_keyframes, all_preprocessed, patch_size, patches_per_frame,
fg_proportion)
else:
patches = (None, None)
patches_images, patches_labels = patches
# generate features
print("\nGenerating features ...", flush=True)
all_features = []
with ProcessPoolExecutor(max_workers=feature_workers) as executor:
for lect_idx, lecture_images in enumerate(patches_images):
print("Processing patches from lecture {0:d} out of {1:d}".format(lect_idx + 1, len(patches_images)))
lecture_features = []
for i, patch_features in enumerate(executor.map(feature_function, lecture_images)):
lecture_features.append(patch_features)
all_features.append(lecture_features)
print("\nSaving patches and features to file")
out_file = open(patch_filename, "wb")
pickle.dump(patches_labels, out_file, pickle.HIGHEST_PROTOCOL)
pickle.dump(patches_images, out_file, pickle.HIGHEST_PROTOCOL)
pickle.dump(all_features, out_file, pickle.HIGHEST_PROTOCOL)
out_file.close()
else:
# load patches from file ....
print("Loading patches and features from file")
in_file = open(patch_filename, "rb")
patches_labels = pickle.load(in_file)
patches_images = pickle.load(in_file)
all_features = pickle.load(in_file)
in_file.close()
end_patch_extraction = time.time()
total_training_time = 0.0
total_binarization_time = 0.0
total_evaluation_time = 0.0
cross_validated_classifiers = []
if not OTSU_mode:
start_training = time.time()
# train classifier using training patches ...
count_all_patches = sum([len(lecture_images) for lecture_images in patches_images])
print("Total patches available for training: " + str(count_all_patches))
n_features = len(all_features[0][0])
print("Total Features: " + str(n_features))
# check local performance using cross-validation based on leaving one lecture out
conf_matrix = np.zeros((2, 2), dtype=np.int32)
avg_train_accuracy = 0.0
rf_max_features = min(rf_max_features, n_features)
if run_crossvalidation:
for i in range(len(patches_images)):
print("Cross-validation fold #" + str(i + 1))
training_data = []
training_labels = []
testing_data = []
testing_labels = []
for k in range(len(patches_images)):
if i == k:
testing_data += all_features[k]
testing_labels += patches_labels[k]
else:
training_data += all_features[k]
training_labels += patches_labels[k]
training_data = np.array(training_data)
testing_data = np.array(testing_data)
print("-> Training Samples: " + str(training_data.shape[0]))
print("-> Testing Samples: " + str(testing_data.shape[0]))
# classification mode ...
# random forest ...
classifier = RandomForestClassifier(rf_n_trees, max_features=rf_max_features, max_depth=rf_max_depth, n_jobs=-1)
classifier.fit(training_data, training_labels)
# keep reference to the n-th fold classifier
cross_validated_classifiers.append(classifier)
pred_labels = classifier.predict(training_data)
train_conf_matrix = np.zeros((2, 2), dtype=np.int32)
for train_idx in range(len(training_labels)):
train_conf_matrix[training_labels[train_idx], pred_labels[train_idx]] += 1
pixel_accuracy = (train_conf_matrix[0, 0] + train_conf_matrix[1, 1]) / len(training_labels)
print("-> Train pixel accuracy: " + str(pixel_accuracy * 100.0))
avg_train_accuracy += pixel_accuracy
pred_labels = classifier.predict(testing_data)
for test_idx in range(len(testing_labels)):
conf_matrix[testing_labels[test_idx], pred_labels[test_idx]] += 1
pixel_accuracy = (conf_matrix[0, 0] + conf_matrix[1, 1]) / count_all_patches
avg_train_accuracy /= len(all_features)
print("Combined testing confusion matrix: ")
print(conf_matrix)
print("Final training pixel accuracy: " + str(avg_train_accuracy * 100.0))
print("Final testing pixel accuracy: " + str(pixel_accuracy * 100.0))
# now, use all data to train a classifier for binarization of all frames ...
if not os.path.exists(classifier_file) or force_update or retrain_classifier:
print("Training classifier using all patches", flush=True)
# classification
training_data = []
training_labels = []
for k in range(len(patches_images)):
training_data += all_features[k]
training_labels += patches_labels[k]
training_data = np.array(training_data)
# Train Random Forest
classifier = RandomForestClassifier(rf_n_trees, max_features=rf_max_features, max_depth=rf_max_depth, n_jobs=-1)
classifier.fit(training_data, training_labels)
print("Saving classifier to file")
out_file = open(classifier_file, "wb")
pickle.dump(classifier, out_file, pickle.HIGHEST_PROTOCOL)
out_file.close()
else:
print("Loading classifier from file")
in_file = open(classifier_file, "rb")
classifier = pickle.load(in_file)
in_file.close()
# release memory (a lot) of elements that will not be used after this point ...
all_features = None
patches_labels = None
training_data = None
training_labels = None
testing_data = None
testing_labels = None
end_training = time.time()
total_training_time += end_training - start_training
# binarize using parameter combination...
start_binarizing = time.time()
last_lecture = None
lecture_offset = -1
training_set = database.get_dataset("training")
for idx, bin_kf in enumerate(binarized_keyframes):
if bin_kf.lecture != last_lecture:
last_lecture = bin_kf.lecture
lecture_offset += 1
print("binarizing kf #" + str(idx) + ", from " + training_set[lecture_offset].title, end="\r", flush=True)
if OTSU_mode:
# ideal BG removal ...
#strel = cv2.getStructuringElement(cv2.MORPH_RECT, (int(patch_size), int(patch_size)))
#bg_mask = all_keyframes[idx].object_mask > 0
#all_preprocessed[idx][bg_mask] = 0
otsu_t, bin_res = cv2.threshold(all_preprocessed[idx].astype(np.uint8), 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
bin_kf.binary_image = np.zeros((bin_res.shape[0], bin_res.shape[1], 3), dtype=np.uint8)
bin_kf.binary_image[:, :, 0] = 255 - bin_res.copy()
bin_kf.binary_image[:, :, 1] = bin_kf.binary_image[:, :, 0].copy()
bin_kf.binary_image[:, :, 2] = bin_kf.binary_image[:, :, 0].copy()
else:
# set classifier for binarization ....
if run_crossvalidation:
# use the classifier that has not seen this image ...
ml_binarizer.classifier = cross_validated_classifiers[lecture_offset]
else:
# use the globally train classifier
ml_binarizer.classifier = classifier
# ... binarize the pre-processed image ...
binary_image = ml_binarizer.preprocessed_binarize(all_preprocessed[idx])
# Do hystheresis filtering ...
otsu_t, high_bin = cv2.threshold(all_preprocessed[idx].astype(np.uint8), 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)
low_bin = binary_image
filtered_bin = 255 - MLBinarizer.binary_hysteresis(low_bin, high_bin)
bin_kf.binary_image = np.zeros((filtered_bin.shape[0], filtered_bin.shape[1], 3), dtype=np.uint8)
bin_kf.binary_image[:, :, 0] = filtered_bin
bin_kf.binary_image[:, :, 1] = filtered_bin
bin_kf.binary_image[:, :, 2] = filtered_bin
bin_kf.update_binary_cc(False)
if config.get("ML_BINARIZER_SAVE_BINARY", True):
if OTSU_mode:
out_name = "TEMPO_OTSU_baseline_binarized_" + str(idx) + ".png"
else:
out_name = "TEMPO_rf_baseline_binarized_" + str(idx) + ".png"
cv2.imwrite(out_name, bin_kf.binary_image)
end_binarizing = time.time()
total_binarization_time += end_binarizing - start_binarizing
# run evaluation metrics ...
print("Computing final evaluation metrics....")
# Summary level metrics ....
start_evaluation = time.time()
EvalParameters.UniqueCC_global_tran_window = 1
EvalParameters.UniqueCC_min_precision = [0.50, 0.55, 0.60, 0.65, 0.70, 0.75, 0.80, 0.90, 0.95]
EvalParameters.UniqueCC_min_recall = [0.50, 0.55, 0.60, 0.65, 0.70, 0.75, 0.80, 0.90, 0.95]
EvalParameters.Report_Summary_Show_Counts = False
EvalParameters.Report_Summary_Show_AVG_per_frame = False
EvalParameters.Report_Summary_Show_Globals = True
all_scope_metrics, scopes = Evaluator.compute_summary_metrics(fake_segments, all_keyframes, fake_unique_groups,
fake_cc_group, fake_segments, binarized_keyframes,
False)
for scope in scopes:
print("")
print("Metrics for scope: " + scope)
print(" \t \tRecall\t \t \tPrecision")
print("Min R.\tMin P.\tE + P\tE. Only\tP. Only\tE + P\tE. Only\tP. Only\tBG. %\tNo BG P.")
scope_metrics = all_scope_metrics[scope]
recall_percent_row = "{0:.2f}\t{1:.2f}\t{2:.2f}\t{3:.2f}\t{4:.2f}"
prec_percent_row = "{0:.2f}\t{1:.2f}\t{2:.2f}\t{3:.2f}\t{4:.2f}"
for all_metrics in scope_metrics:
metrics = all_metrics["recall_metrics"]
recall_str = recall_percent_row.format(all_metrics["min_cc_recall"] * 100.0,
all_metrics["min_cc_precision"] * 100.0,
metrics["recall"] * 100.0, metrics["only_exact_recall"] * 100.0,
metrics["only_partial_recall"] * 100.0)
metrics = all_metrics["precision_metrics"]
prec_str = prec_percent_row.format(metrics["precision"] * 100.0, metrics["only_exact_precision"] * 100.0,
metrics["only_partial_precision"] * 100.0,
metrics["global_bg_unmatched"] * 100.0,
metrics["no_bg_precision"] * 100.0)
print(recall_str + "\t" + prec_str)
# pixel level metrics
pixel_metrics = Evaluator.compute_pixel_binary_metrics(all_keyframes, binarized_keyframes)
print("Pixel level metrics")
for key in sorted(pixel_metrics.keys()):
print("{0:s}\t{1:.2f}".format(key, pixel_metrics[key] *100.0))
end_evaluation = time.time()
total_evaluation_time += end_evaluation - start_evaluation
end_everything = time.time()
print("Total loading time: " + TimeHelper.secondsToStr(end_loading - start_loading))
print("Total preprocessing time: " + TimeHelper.secondsToStr(end_preprocessing - start_preprocessing))
print("Total patch extraction time: " + TimeHelper.secondsToStr(end_patch_extraction - start_patch_extraction))
print("Total training time: " + TimeHelper.secondsToStr(total_training_time))
print("Total binarization time: " + TimeHelper.secondsToStr(total_binarization_time))
print("Total evaluation time: " + TimeHelper.secondsToStr(total_evaluation_time))
print("Total Time: " + TimeHelper.secondsToStr(end_everything - start_loading))
def __init__(self, size, db_name, lecture_title, output_path):
Screen.__init__(self, "Unique CC Ground Truth Annotation Interface", size)
general_background = (100, 90, 80)
text_color = (255, 255, 255)
button_text_color = (35, 50, 20)
button_back_color = (228, 228, 228)
self.elements.back_color = general_background
self.db_name = db_name
self.lecture_title = lecture_title
self.output_path = output_path
export_filename = self.output_path + "/segments.xml"
export_image_prefix = self.output_path + "/keyframes/"
# load including segment information
self.keyframe_annotations, self.segments = KeyFrameAnnotation.LoadExportedKeyframes(export_filename, export_image_prefix, True)
if len(self.keyframe_annotations) > 0:
print("Key-frames Loaded: " + str(len(self.keyframe_annotations)))
else:
raise Exception("Cannot start with 0 key-frames")
portions_filename = self.output_path + "/portions.xml"
portions_path = self.output_path + "/portions/"
if os.path.exists(portions_filename):
# Saved data detected, loading
print("Previously saved portion data detected, loading")
KeyFrameAnnotation.LoadKeyframesPortions(portions_filename, self.keyframe_annotations, portions_path)
else:
raise Exception("No saved portion data detected, cannot continue")
print("Original Key-frames: " + str(len(self.keyframe_annotations)))
print("Segments: " + str(len(self.segments)))
self.keyframe_annotations = KeyFrameAnnotation.CombineKeyframesPerSegment(self.keyframe_annotations, self.segments, True)
print("Key-frames after combination per segment" + str(len(self.keyframe_annotations)))
# other CC/group elements
self.unique_groups = None
self.cc_group = None
self.colored_cache = []
self.cc_total = 0
for kf_idx, keyframe in enumerate(self.keyframe_annotations):
self.cc_total += len(keyframe.binary_cc)
unique_cc_filename = self.output_path + "/unique_ccs.xml"
if os.path.exists(unique_cc_filename):
# Saved data detected, loading
print("Previously saved unique CC data detected, loading")
self.cc_group, self.unique_groups = UniqueCCGroup.GroupsFromXML(self.keyframe_annotations, unique_cc_filename)
else:
# no previous data, build default index (all CCs are unique)
self.unique_groups = []
self.cc_group = []
for kf_idx, keyframe in enumerate(self.keyframe_annotations):
self.cc_group.append({})
for cc in keyframe.binary_cc:
new_group = UniqueCCGroup(cc, kf_idx)
self.unique_groups.append(new_group)
self.cc_group[kf_idx][cc.strID()] = new_group
self.update_colored_cache(0)
self.view_mode = GTUniqueCCAnnotator.ViewModeColored
self.edition_mode = GTUniqueCCAnnotator.ModeNavigate
self.view_scale = 1.0
self.selected_keyframe = 0
self.matching_delta_x = 0
self.matching_delta_y = 0
self.matching_scores = None
self.matching_min_recall = 0.99
self.matching_min_precision = 0.99
self.base_matching = None
# add elements....
container_top = 10
container_width = 330
button_2_width = 150
button_2_left = int(container_width * 0.25) - button_2_width / 2
button_2_right = int(container_width * 0.75) - button_2_width / 2
# Navigation panel to move accross frames
self.container_nav_buttons = ScreenContainer("container_nav_buttons", (container_width, 70), back_color=general_background)
self.container_nav_buttons.position = (self.width - self.container_nav_buttons.width - 10, container_top)
self.elements.append(self.container_nav_buttons)
self.lbl_nav_keyframe = ScreenLabel("lbl_nav_keyframe", "Key-Frame: 1 / " + str(len(self.keyframe_annotations)), 21, 290, 1)
self.lbl_nav_keyframe.position = (5, 5)
self.lbl_nav_keyframe.set_background(general_background)
self.lbl_nav_keyframe.set_color(text_color)
self.container_nav_buttons.append(self.lbl_nav_keyframe)
time_str = TimeHelper.stampToStr(self.keyframe_annotations[self.selected_keyframe].time)
self.lbl_nav_time = ScreenLabel("lbl_nav_time", time_str, 21, 290, 1)
self.lbl_nav_time.position = (5, self.lbl_nav_keyframe.get_bottom() + 20)
self.lbl_nav_time.set_background(general_background)
self.lbl_nav_time.set_color(text_color)
self.container_nav_buttons.append(self.lbl_nav_time)
self.btn_nav_keyframe_prev = ScreenButton("btn_nav_keyframe_prev", "Prev", 21, 90)
self.btn_nav_keyframe_prev.set_colors(button_text_color, button_back_color)
self.btn_nav_keyframe_prev.position = (10, self.lbl_nav_keyframe.get_bottom() + 10)
self.btn_nav_keyframe_prev.click_callback = self.btn_nav_keyframe_prev_click
self.container_nav_buttons.append(self.btn_nav_keyframe_prev)
self.btn_nav_keyframe_next = ScreenButton("btn_nav_keyframe_next", "Next", 21, 90)
self.btn_nav_keyframe_next.set_colors(button_text_color, button_back_color)
self.btn_nav_keyframe_next.position = (self.container_nav_buttons.width - self.btn_nav_keyframe_next.width - 10,
self.lbl_nav_keyframe.get_bottom() + 10)
self.btn_nav_keyframe_next.click_callback = self.btn_nav_keyframe_next_click
self.container_nav_buttons.append(self.btn_nav_keyframe_next)
# confirmation panel
self.container_confirm_buttons = ScreenContainer("container_confirm_buttons", (container_width, 70), back_color=general_background)
self.container_confirm_buttons.position = (self.width - self.container_confirm_buttons.width - 10, container_top)
self.elements.append(self.container_confirm_buttons)
self.container_confirm_buttons.visible = False
self.lbl_confirm_message = ScreenLabel("lbl_confirm_message", "Confirmation message goes here?", 21, 290, 1)
self.lbl_confirm_message.position = (5, 5)
self.lbl_confirm_message.set_background(general_background)
self.lbl_confirm_message.set_color(text_color)
self.container_confirm_buttons.append(self.lbl_confirm_message)
self.btn_confirm_cancel = ScreenButton("btn_confirm_cancel", "Cancel", 21, 130)
self.btn_confirm_cancel.set_colors(button_text_color, button_back_color)
self.btn_confirm_cancel.position = (10, self.lbl_nav_keyframe.get_bottom() + 10)
self.btn_confirm_cancel.click_callback = self.btn_confirm_cancel_click
self.container_confirm_buttons.append(self.btn_confirm_cancel)
self.btn_confirm_accept = ScreenButton("btn_confirm_accept", "Accept", 21, 130)
self.btn_confirm_accept.set_colors(button_text_color, button_back_color)
self.btn_confirm_accept.position = (self.container_confirm_buttons.width - self.btn_confirm_accept.width - 10,
self.lbl_confirm_message.get_bottom() + 10)
self.btn_confirm_accept.click_callback = self.btn_confirm_accept_click
self.container_confirm_buttons.append(self.btn_confirm_accept)
# View panel with view control buttons
self.container_view_buttons = ScreenContainer("container_view_buttons", (container_width, 165), back_color=general_background)
self.container_view_buttons.position = (self.width - self.container_view_buttons.width - 10,
self.container_nav_buttons.get_bottom() + 10)
self.elements.append(self.container_view_buttons)
button_width = 190
button_left = (self.container_view_buttons.width - button_width) / 2
# zoom ....
self.lbl_zoom = ScreenLabel("lbl_zoom", "Zoom: 100%", 21, container_width - 10, 1)
self.lbl_zoom.position = (5, 5)
self.lbl_zoom.set_background(general_background)
self.lbl_zoom.set_color(text_color)
self.container_view_buttons.append(self.lbl_zoom)
self.btn_zoom_reduce = ScreenButton("btn_zoom_reduce", "[ - ]", 21, 90)
self.btn_zoom_reduce.set_colors(button_text_color, button_back_color)
self.btn_zoom_reduce.position = (10, self.lbl_zoom.get_bottom() + 10)
self.btn_zoom_reduce.click_callback = self.btn_zoom_reduce_click
self.container_view_buttons.append(self.btn_zoom_reduce)
self.btn_zoom_increase = ScreenButton("btn_zoom_increase", "[ + ]", 21, 90)
self.btn_zoom_increase.set_colors(button_text_color, button_back_color)
self.btn_zoom_increase.position = (self.container_view_buttons.width - self.btn_zoom_increase.width - 10,
self.lbl_zoom.get_bottom() + 10)
self.btn_zoom_increase.click_callback = self.btn_zoom_increase_click
self.container_view_buttons.append(self.btn_zoom_increase)
self.btn_zoom_zero = ScreenButton("btn_zoom_zero", "100%", 21, 90)
self.btn_zoom_zero.set_colors(button_text_color, button_back_color)
self.btn_zoom_zero.position = ((self.container_view_buttons.width - self.btn_zoom_zero.width) / 2,
self.lbl_zoom.get_bottom() + 10)
self.btn_zoom_zero.click_callback = self.btn_zoom_zero_click
self.container_view_buttons.append(self.btn_zoom_zero)
self.btn_view_raw = ScreenButton("btn_view_raw", "Raw View", 21, button_2_width)
self.btn_view_raw.set_colors(button_text_color, button_back_color)
self.btn_view_raw.position = (button_2_left, self.btn_zoom_zero.get_bottom() + 10)
self.btn_view_raw.click_callback = self.btn_view_raw_click
self.container_view_buttons.append(self.btn_view_raw)
self.btn_view_gray = ScreenButton("btn_view_gray", "Grayscale View", 21, button_2_width)
self.btn_view_gray.set_colors(button_text_color, button_back_color)
self.btn_view_gray.position = (button_2_right, self.btn_zoom_zero.get_bottom() + 10)
self.btn_view_gray.click_callback = self.btn_view_gray_click
self.container_view_buttons.append(self.btn_view_gray)
self.btn_view_binary = ScreenButton("btn_view_binary", "Binary View", 21, button_2_width)
self.btn_view_binary.set_colors(button_text_color, button_back_color)
self.btn_view_binary.position = (button_2_left, self.btn_view_gray.get_bottom() + 10)
self.btn_view_binary.click_callback = self.btn_view_binary_click
self.container_view_buttons.append(self.btn_view_binary)
self.btn_view_colored = ScreenButton("btn_view_colored", "Colored View", 21, button_2_width)
self.btn_view_colored.set_colors(button_text_color, button_back_color)
self.btn_view_colored.position = (button_2_right, self.btn_view_gray.get_bottom() + 10)
self.btn_view_colored.click_callback = self.btn_view_colored_click
self.container_view_buttons.append(self.btn_view_colored)
# Panel with action buttons (Add/Remove links)
self.container_action_buttons = ScreenContainer("container_action_buttons", (container_width, 45),
general_background)
self.container_action_buttons.position = (self.container_view_buttons.get_left(), self.container_view_buttons.get_bottom() + 5)
self.elements.append(self.container_action_buttons)
self.btn_matches_add = ScreenButton("btn_matches_add", "Add Matches", 21, button_2_width)
self.btn_matches_add.set_colors(button_text_color, button_back_color)
self.btn_matches_add.position = (button_2_left, 5)
self.btn_matches_add.click_callback = self.btn_matches_add_click
self.container_action_buttons.append(self.btn_matches_add)
self.btn_matches_del = ScreenButton("btn_matches_del", "Del. Matches", 21, button_2_width)
self.btn_matches_del.set_colors(button_text_color, button_back_color)
self.btn_matches_del.position = (button_2_right, 5)
self.btn_matches_del.click_callback = self.btn_matches_del_click
self.container_action_buttons.append(self.btn_matches_del)
# ===============================================
# Panel with matching parameters for step 1 (Matching Translation)
self.container_matching_translation = ScreenContainer("container_matching_translation", (container_width, 150), general_background)
self.container_matching_translation.position = (self.container_view_buttons.get_left(), self.container_view_buttons.get_bottom() + 5)
self.elements.append(self.container_matching_translation)
self.lbl_translation_title = ScreenLabel("lbl_translation_title", "Translation Parameters", 21, container_width - 10, 1)
self.lbl_translation_title.position = (5, 5)
self.lbl_translation_title.set_background(general_background)
self.lbl_translation_title.set_color(text_color)
self.container_matching_translation.append(self.lbl_translation_title)
self.lbl_delta_x = ScreenLabel("lbl_delta_x", "Delta X: " + str(self.matching_delta_x), 21, container_width - 10, 1)
self.lbl_delta_x.position = (5, self.lbl_translation_title.get_bottom() + 20)
self.lbl_delta_x.set_background(general_background)
self.lbl_delta_x.set_color(text_color)
self.container_matching_translation.append(self.lbl_delta_x)
max_delta = GTUniqueCCAnnotator.ParamsMaxTranslation
self.scroll_delta_x = ScreenHorizontalScroll("scroll_delta_x", -max_delta, max_delta, 0, 1)
self.scroll_delta_x.position = (5, self.lbl_delta_x.get_bottom() + 10)
self.scroll_delta_x.width = container_width - 10
self.scroll_delta_x.scroll_callback = self.scroll_delta_x_change
self.container_matching_translation.append(self.scroll_delta_x)
self.lbl_delta_y = ScreenLabel("lbl_delta_y", "Delta Y: " + str(self.matching_delta_y), 21, container_width - 10, 1)
self.lbl_delta_y.position = (5, self.scroll_delta_x.get_bottom() + 20)
self.lbl_delta_y.set_background(general_background)
self.lbl_delta_y.set_color(text_color)
self.container_matching_translation.append(self.lbl_delta_y)
self.scroll_delta_y = ScreenHorizontalScroll("scroll_delta_y", -max_delta, max_delta, 0, 1)
self.scroll_delta_y.position = (5, self.lbl_delta_y.get_bottom() + 10)
self.scroll_delta_y.width = container_width - 10
self.scroll_delta_y.scroll_callback = self.scroll_delta_y_change
self.container_matching_translation.append(self.scroll_delta_y)
self.container_matching_translation.visible = False
# ===============================================
# Panel with matching parameters for step 2 (Matching Strictness)
self.container_matching_strictness = ScreenContainer("container_matching_strictness", (container_width, 150),
general_background)
self.container_matching_strictness.position = (self.container_view_buttons.get_left(), self.container_view_buttons.get_bottom() + 5)
self.elements.append(self.container_matching_strictness)
self.lbl_matching_title = ScreenLabel("lbl_matching_title", "Matching Parameters", 21,
container_width - 10, 1)
self.lbl_matching_title.position = (5, 5)
self.lbl_matching_title.set_background(general_background)
self.lbl_matching_title.set_color(text_color)
self.container_matching_strictness.append(self.lbl_matching_title)
str_recall = "Minimum Recall: " + str(int(self.matching_min_recall * 100))
self.lbl_min_recall = ScreenLabel("lbl_min_recall", str_recall, 21, container_width - 10, 1)
self.lbl_min_recall.position = (5, self.lbl_matching_title.get_bottom() + 20)
self.lbl_min_recall.set_background(general_background)
self.lbl_min_recall.set_color(text_color)
self.container_matching_strictness.append(self.lbl_min_recall)
min_recall = GTUniqueCCAnnotator.ParamsMinRecall
self.scroll_min_recall = ScreenHorizontalScroll("scroll_min_recall", min_recall, 100, 99, 1)
self.scroll_min_recall.position = (5, self.lbl_min_recall.get_bottom() + 10)
self.scroll_min_recall.width = container_width - 10
self.scroll_min_recall.scroll_callback = self.scroll_min_recall_change
self.container_matching_strictness.append(self.scroll_min_recall)
str_precision = "Minimum Precision: " + str(int(self.matching_min_precision * 100))
self.lbl_min_precision = ScreenLabel("lbl_min_precision", str_precision, 21, container_width - 10, 1)
self.lbl_min_precision.position = (5, self.scroll_min_recall.get_bottom() + 20)
self.lbl_min_precision.set_background(general_background)
self.lbl_min_precision.set_color(text_color)
self.container_matching_strictness.append(self.lbl_min_precision)
min_precision = GTUniqueCCAnnotator.ParamsMinPrecision
self.scroll_min_precision = ScreenHorizontalScroll("scroll_min_precision", min_precision, 100, 99, 1)
self.scroll_min_precision.position = (5, self.lbl_min_precision.get_bottom() + 10)
self.scroll_min_precision.width = container_width - 10
self.scroll_min_precision.scroll_callback = self.scroll_min_precision_change
self.container_matching_strictness.append(self.scroll_min_precision)
self.container_matching_strictness.visible = False
# ===============================================
stats_background = (60, 50, 40)
self.container_stats = ScreenContainer("container_stats", (container_width, 70), back_color=stats_background)
self.container_stats.position = (self.width - container_width - 10, self.container_action_buttons.get_bottom() + 5)
self.elements.append(self.container_stats)
self.lbl_cc_stats = ScreenLabel("lbl_cc_stats", "Connected Component Stats", 21, container_width - 10, 1)
self.lbl_cc_stats.position = (5, 5)
self.lbl_cc_stats.set_background(stats_background)
self.lbl_cc_stats.set_color(text_color)
self.container_stats.append(self.lbl_cc_stats)
self.lbl_cc_raw = ScreenLabel("lbl_cc_raw", "Total Raw CC:\n" + str(self.cc_total), 21, button_2_width, 1)
self.lbl_cc_raw.position = (button_2_left, self.lbl_cc_stats.get_bottom() + 10)
self.lbl_cc_raw.set_background(stats_background)
self.lbl_cc_raw.set_color(text_color)
self.container_stats.append(self.lbl_cc_raw)
self.lbl_cc_unique = ScreenLabel("lbl_cc_unique", "Total Unique CC:\n" + str(len(self.unique_groups)), 21, button_2_width, 1)
self.lbl_cc_unique.position = (button_2_right, self.lbl_cc_stats.get_bottom() + 10)
self.lbl_cc_unique.set_background(stats_background)
self.lbl_cc_unique.set_color(text_color)
self.container_stats.append(self.lbl_cc_unique)
#=============================================================
# Panel with state buttons (Undo, Redo, Save)
self.container_state_buttons = ScreenContainer("container_state_buttons", (container_width, 200),
general_background)
self.container_state_buttons.position = (
self.container_view_buttons.get_left(), self.container_stats.get_bottom() + 10)
self.elements.append(self.container_state_buttons)
self.btn_undo = ScreenButton("btn_undo", "Undo", 21, button_width)
self.btn_undo.set_colors(button_text_color, button_back_color)
self.btn_undo.position = (button_left, 5)
self.btn_undo.click_callback = self.btn_undo_click
self.container_state_buttons.append(self.btn_undo)
self.btn_redo = ScreenButton("btn_redo", "Redo", 21, button_width)
self.btn_redo.set_colors(button_text_color, button_back_color)
self.btn_redo.position = (button_left, self.btn_undo.get_bottom() + 10)
self.btn_redo.click_callback = self.btn_redo_click
self.container_state_buttons.append(self.btn_redo)
self.btn_save = ScreenButton("btn_save", "Save", 21, button_width)
self.btn_save.set_colors(button_text_color, button_back_color)
self.btn_save.position = (button_left, self.btn_redo.get_bottom() + 10)
self.btn_save.click_callback = self.btn_save_click
self.container_state_buttons.append(self.btn_save)
self.btn_exit = ScreenButton("btn_exit", "Exit", 21, button_width)
self.btn_exit.set_colors(button_text_color, button_back_color)
self.btn_exit.position = (button_left, self.btn_save.get_bottom() + 30)
self.btn_exit.click_callback = self.btn_exit_click
self.container_state_buttons.append(self.btn_exit)
# print("MAKE CONTAINER STATS VISIBLE AGAIN!!!")
# self.container_stats.visible = False
# self.container_state_buttons.visible = False
# ==============================================================
image_width = self.width - self.container_nav_buttons.width - 30
image_height = self.height - container_top - 10
self.container_images = ScreenContainer("container_images", (image_width, image_height), back_color=(0, 0, 0))
self.container_images.position = (10, container_top)
self.elements.append(self.container_images)
# ... image objects ...
tempo_blank = np.zeros((50, 50, 3), np.uint8)
tempo_blank[:, :, :] = 255
self.img_main = ScreenImage("img_main", tempo_blank, 0, 0, True, cv2.INTER_NEAREST)
self.img_main.position = (0, 0)
#self.img_main.mouse_button_down_callback = self.img_mouse_down
self.container_images.append(self.img_main)
# canvas used for annotations
self.canvas_select = ScreenCanvas("canvas_select", 100, 100)
self.canvas_select.position = (0, 0)
self.canvas_select.locked = False
# self.canvas_select.object_edited_callback = self.canvas_object_edited
# self.canvas_select.object_selected_callback = self.canvas_selection_changed
self.container_images.append(self.canvas_select)
self.canvas_select.add_element("selection_rectangle", 10, 10, 40, 40)
self.canvas_select.elements["selection_rectangle"].visible = False
self.undo_stack = []
self.redo_stack = []
self.update_current_view(True)