def btn_save_click(self, button): xml_str = UniqueCCGroup.GenerateGroupsXML(self.keyframe_annotations, self.unique_groups) unique_cc_filename = self.output_path + "/unique_ccs.xml" out_file = open(unique_cc_filename, "w") out_file.write(xml_str) out_file.close() print("Saved to: " + unique_cc_filename)
def generate_fake_keyframe_info(all_keyframes): # here, we use fake unique CC's as we care to improve per-frame recall/precision fake_unique_groups = [] fake_cc_group = [] fake_segments = [] for kf_idx, keyframe in enumerate(all_keyframes): fake_segments.append((kf_idx * 5 + 1, kf_idx * 5 + 4)) fake_cc_group.append({}) for cc in keyframe.binary_cc: new_group = UniqueCCGroup(cc, kf_idx) fake_unique_groups.append(new_group) fake_cc_group[kf_idx][cc.strID()] = new_group return fake_unique_groups, fake_cc_group, fake_segments
def process_summary(self, process, input_data): database = process.database lecture = process.current_lecture if "b" in process.params: base_line_prefix = process.params["b"] + "_" else: base_line_prefix = "" lecture_suffix = database.name + "_" + lecture.title.lower() summary_prefix = database.output_summaries + "/" + base_line_prefix + lecture_suffix annotation_prefix = database.output_annotations + "/" + lecture_suffix annot_filename = annotation_prefix + "/segments.xml" annot_cc_groups_filename = annotation_prefix + "/unique_ccs.xml" annot_image_prefix = annotation_prefix + "/keyframes/" annot_binary_prefix = annotation_prefix + "/binary/" print("-> loading data ...") start_loading = time.time() # ideal summary ... annot_keyframes, annot_segments = KeyFrameAnnotation.LoadExportedKeyframes( annot_filename, annot_image_prefix, True) for keyframe in annot_keyframes: keyframe.binary_image = cv2.imread(annot_binary_prefix + str(keyframe.idx) + ".png") keyframe.update_binary_cc(False) annot_keyframes = KeyFrameAnnotation.CombineKeyframesPerSegment( annot_keyframes, annot_segments, False) annot_cc_group, annot_unique_groups = UniqueCCGroup.GroupsFromXML( annot_keyframes, annot_cc_groups_filename) # provided summary ... summ_filename = summary_prefix + "/segments.xml" summ_image_prefix = summary_prefix + "/keyframes/" summ_keyframes, summ_segments = KeyFrameAnnotation.LoadExportedKeyframes( summ_filename, summ_image_prefix, True, False, True) for keyframe in summ_keyframes: # keyframe.binary_image = keyframe.raw_image.copy() keyframe.update_binary_cc(False) summ_keyframes = KeyFrameAnnotation.CombineKeyframesPerSegment( summ_keyframes, summ_segments, False) print("-> data loaded!") print("-> computing metrics ...") # TODO: This should come from configuration # output_prefix = database.output_evaluation + "/" + base_line_prefix + database.name + "_" + lecture.title.lower() output_prefix = "output/evaluation" + "/" + base_line_prefix + lecture_suffix # compute metrics and store matching results as images ... EvalParameters.Report_Summary_Show_stats_per_size = True all_metrics, ranges = Evaluator.compute_summary_metrics( annot_segments, annot_keyframes, annot_unique_groups, annot_cc_group, summ_segments, summ_keyframes, False, output_prefix) self.per_lecture_metrics[lecture.title] = all_metrics self.total_per_lecture_keyframes[lecture.title] = len(summ_keyframes) self.ranges_per_lecture[lecture.title] = ranges
def __init__(self, size, db_name, lecture_title, output_path): Screen.__init__(self, "Formula Ground Truth Annotation Interface", size) general_background = (50, 80, 160) text_color = (255, 255, 255) button_text_color = (20, 20, 50) button_back_color = (228, 228, 228) self.elements.back_color = general_background self.db_name = db_name self.lecture_title = lecture_title self.output_path = output_path export_filename = self.output_path + "/segments.xml" export_image_prefix = self.output_path + "/keyframes/" # load including segment information self.keyframe_annotations, self.segments = KeyFrameAnnotation.LoadExportedKeyframes( export_filename, export_image_prefix, True) if len(self.keyframe_annotations) > 0: print("Key-frames Loaded: " + str(len(self.keyframe_annotations))) else: raise Exception("Cannot start with 0 key-frames") portions_filename = self.output_path + "/portions.xml" portions_path = self.output_path + "/portions/" if os.path.exists(portions_filename): # Saved data detected, loading print("Previously saved portion data detected, loading") KeyFrameAnnotation.LoadKeyframesPortions(portions_filename, self.keyframe_annotations, portions_path) else: raise Exception("No saved portion data detected, cannot continue") print("Original Key-frames: " + str(len(self.keyframe_annotations))) print("Segments: " + str(len(self.segments))) self.keyframe_annotations = KeyFrameAnnotation.CombineKeyframesPerSegment( self.keyframe_annotations, self.segments, True) print("Key-frames after combination per segment" + str(len(self.keyframe_annotations))) # other CC/group elements self.unique_groups = None self.cc_group = None self.cc_total = 0 for kf_idx, keyframe in enumerate(self.keyframe_annotations): self.cc_total += len(keyframe.binary_cc) unique_cc_filename = self.output_path + "/unique_ccs.xml" if os.path.exists(unique_cc_filename): # Saved data detected, loading print("Previously saved unique CC data detected, loading") self.cc_group, self.unique_groups = UniqueCCGroup.GroupsFromXML( self.keyframe_annotations, unique_cc_filename) else: # no previous data, build default index (all CCs are unique) raise Exception( "No unique CC data found for lecture. Must label Unique CC first" ) self.view_mode = GTFormulaAnnotator.ViewModeColored self.edition_mode = GTFormulaAnnotator.ModeNavigate self.view_scale = 1.0 self.selected_keyframe = 0 self.selected_formula = 0 self.adding_groups = [] self.formulas_per_frame = [ [] for idx in range(len(self.keyframe_annotations)) ] saved_filename = self.output_path + "/formula_ccs.xml" if os.path.exists(saved_filename): # load saved data ... self.formulas_ccs = FormulaCCs.FormulasFromXML( self.unique_groups, saved_filename) # add to index per frame ... for formula in self.formulas_ccs: for frame_idx in range(formula.first_visible, formula.last_visible + 1): self.formulas_per_frame[frame_idx].append(formula) print("Loaded: " + saved_filename) else: self.formulas_ccs = [] # update draw cache of highlighted formulae ... self.colored_cache = [None] * len(self.keyframe_annotations) for idx in range(len(self.keyframe_annotations)): self.update_colored_cache(idx) # add elements.... container_top = 10 container_width = 330 button_2_width = 150 button_2_left = int(container_width * 0.25) - button_2_width / 2 button_2_right = int(container_width * 0.75) - button_2_width / 2 # Navigation panel to move accross frames self.container_nav_buttons = ScreenContainer( "container_nav_buttons", (container_width, 70), back_color=general_background) self.container_nav_buttons.position = ( self.width - self.container_nav_buttons.width - 10, container_top) self.elements.append(self.container_nav_buttons) self.lbl_nav_keyframe = ScreenLabel( "lbl_nav_keyframe", "Key-Frame: 1 / " + str(len(self.keyframe_annotations)), 21, 290, 1) self.lbl_nav_keyframe.position = (5, 5) self.lbl_nav_keyframe.set_background(general_background) self.lbl_nav_keyframe.set_color(text_color) self.container_nav_buttons.append(self.lbl_nav_keyframe) time_str = TimeHelper.stampToStr( self.keyframe_annotations[self.selected_keyframe].time) self.lbl_nav_time = ScreenLabel("lbl_nav_time", time_str, 21, 290, 1) self.lbl_nav_time.position = (5, self.lbl_nav_keyframe.get_bottom() + 20) self.lbl_nav_time.set_background(general_background) self.lbl_nav_time.set_color(text_color) self.container_nav_buttons.append(self.lbl_nav_time) self.btn_nav_keyframe_prev = ScreenButton("btn_nav_keyframe_prev", "Prev", 21, 90) self.btn_nav_keyframe_prev.set_colors(button_text_color, button_back_color) self.btn_nav_keyframe_prev.position = ( 10, self.lbl_nav_keyframe.get_bottom() + 10) self.btn_nav_keyframe_prev.click_callback = self.btn_nav_keyframe_prev_click self.container_nav_buttons.append(self.btn_nav_keyframe_prev) self.btn_nav_keyframe_next = ScreenButton("btn_nav_keyframe_next", "Next", 21, 90) self.btn_nav_keyframe_next.set_colors(button_text_color, button_back_color) self.btn_nav_keyframe_next.position = ( self.container_nav_buttons.width - self.btn_nav_keyframe_next.width - 10, self.lbl_nav_keyframe.get_bottom() + 10) self.btn_nav_keyframe_next.click_callback = self.btn_nav_keyframe_next_click self.container_nav_buttons.append(self.btn_nav_keyframe_next) # ================================================================================ # confirmation panel self.container_confirm_buttons = ScreenContainer( "container_confirm_buttons", (container_width, 70), back_color=general_background) self.container_confirm_buttons.position = ( self.width - self.container_confirm_buttons.width - 10, container_top) self.elements.append(self.container_confirm_buttons) self.container_confirm_buttons.visible = False self.lbl_confirm_message = ScreenLabel( "lbl_confirm_message", "Confirmation message goes here?", 21, 290, 1) self.lbl_confirm_message.position = (5, 5) self.lbl_confirm_message.set_background(general_background) self.lbl_confirm_message.set_color(text_color) self.container_confirm_buttons.append(self.lbl_confirm_message) self.btn_confirm_cancel = ScreenButton("btn_confirm_cancel", "Cancel", 21, 130) self.btn_confirm_cancel.set_colors(button_text_color, button_back_color) self.btn_confirm_cancel.position = ( 10, self.lbl_nav_keyframe.get_bottom() + 10) self.btn_confirm_cancel.click_callback = self.btn_confirm_cancel_click self.container_confirm_buttons.append(self.btn_confirm_cancel) self.btn_confirm_accept = ScreenButton("btn_confirm_accept", "Accept", 21, 130) self.btn_confirm_accept.set_colors(button_text_color, button_back_color) self.btn_confirm_accept.position = ( self.container_confirm_buttons.width - self.btn_confirm_accept.width - 10, self.lbl_confirm_message.get_bottom() + 10) self.btn_confirm_accept.click_callback = self.btn_confirm_accept_click self.container_confirm_buttons.append(self.btn_confirm_accept) # ================================================================================ # View panel with view control buttons self.container_view_buttons = ScreenContainer( "container_view_buttons", (container_width, 165), back_color=general_background) self.container_view_buttons.position = ( self.width - self.container_view_buttons.width - 10, self.container_nav_buttons.get_bottom() + 10) self.elements.append(self.container_view_buttons) button_width = 190 button_left = (self.container_view_buttons.width - button_width) / 2 # zoom .... self.lbl_zoom = ScreenLabel("lbl_zoom", "Zoom: 100%", 21, container_width - 10, 1) self.lbl_zoom.position = (5, 5) self.lbl_zoom.set_background(general_background) self.lbl_zoom.set_color(text_color) self.container_view_buttons.append(self.lbl_zoom) self.btn_zoom_reduce = ScreenButton("btn_zoom_reduce", "[ - ]", 21, 90) self.btn_zoom_reduce.set_colors(button_text_color, button_back_color) self.btn_zoom_reduce.position = (10, self.lbl_zoom.get_bottom() + 10) self.btn_zoom_reduce.click_callback = self.btn_zoom_reduce_click self.container_view_buttons.append(self.btn_zoom_reduce) self.btn_zoom_increase = ScreenButton("btn_zoom_increase", "[ + ]", 21, 90) self.btn_zoom_increase.set_colors(button_text_color, button_back_color) self.btn_zoom_increase.position = (self.container_view_buttons.width - self.btn_zoom_increase.width - 10, self.lbl_zoom.get_bottom() + 10) self.btn_zoom_increase.click_callback = self.btn_zoom_increase_click self.container_view_buttons.append(self.btn_zoom_increase) self.btn_zoom_zero = ScreenButton("btn_zoom_zero", "100%", 21, 90) self.btn_zoom_zero.set_colors(button_text_color, button_back_color) self.btn_zoom_zero.position = ( (self.container_view_buttons.width - self.btn_zoom_zero.width) / 2, self.lbl_zoom.get_bottom() + 10) self.btn_zoom_zero.click_callback = self.btn_zoom_zero_click self.container_view_buttons.append(self.btn_zoom_zero) self.btn_view_raw = ScreenButton("btn_view_raw", "Raw View", 21, button_2_width) self.btn_view_raw.set_colors(button_text_color, button_back_color) self.btn_view_raw.position = (button_2_left, self.btn_zoom_zero.get_bottom() + 10) self.btn_view_raw.click_callback = self.btn_view_raw_click self.container_view_buttons.append(self.btn_view_raw) self.btn_view_gray = ScreenButton("btn_view_gray", "Grayscale View", 21, button_2_width) self.btn_view_gray.set_colors(button_text_color, button_back_color) self.btn_view_gray.position = (button_2_right, self.btn_zoom_zero.get_bottom() + 10) self.btn_view_gray.click_callback = self.btn_view_gray_click self.container_view_buttons.append(self.btn_view_gray) self.btn_view_binary = ScreenButton("btn_view_binary", "Binary View", 21, button_2_width) self.btn_view_binary.set_colors(button_text_color, button_back_color) self.btn_view_binary.position = (button_2_left, self.btn_view_gray.get_bottom() + 10) self.btn_view_binary.click_callback = self.btn_view_binary_click self.container_view_buttons.append(self.btn_view_binary) self.btn_view_colored = ScreenButton("btn_view_colored", "Colored View", 21, button_2_width) self.btn_view_colored.set_colors(button_text_color, button_back_color) self.btn_view_colored.position = (button_2_right, self.btn_view_gray.get_bottom() + 10) self.btn_view_colored.click_callback = self.btn_view_colored_click self.container_view_buttons.append(self.btn_view_colored) # ================================================================================= # Panel with action buttons (Add/Remove links) self.container_action_buttons = ScreenContainer( "container_action_buttons", (container_width, 240), general_background) self.container_action_buttons.position = ( self.container_view_buttons.get_left(), self.container_view_buttons.get_bottom() + 5) self.elements.append(self.container_action_buttons) self.lbl_nav_formula = ScreenLabel("lbl_nav_formula", "X / X", 21, container_width - 10, 1) self.lbl_nav_formula.position = (5, 5) self.lbl_nav_formula.set_background(general_background) self.lbl_nav_formula.set_color(text_color) self.container_action_buttons.append(self.lbl_nav_formula) self.btn_nav_formula_prev = ScreenButton("btn_nav_formula_prev", "Prev", 21, button_2_width) self.btn_nav_formula_prev.set_colors(button_text_color, button_back_color) self.btn_nav_formula_prev.position = ( button_2_left, self.lbl_nav_formula.get_bottom() + 10) self.btn_nav_formula_prev.click_callback = self.btn_nav_formula_prev_click self.container_action_buttons.append(self.btn_nav_formula_prev) self.btn_nav_formula_next = ScreenButton("btn_nav_formula_next", "Next", 21, button_2_width) self.btn_nav_formula_next.set_colors(button_text_color, button_back_color) self.btn_nav_formula_next.position = ( button_2_right, self.lbl_nav_formula.get_bottom() + 10) self.btn_nav_formula_next.click_callback = self.btn_nav_formula_next_click self.container_action_buttons.append(self.btn_nav_formula_next) self.btn_formulas_add = ScreenButton("btn_formulas_add", "Add Formula", 21, button_2_width) self.btn_formulas_add.set_colors(button_text_color, button_back_color) self.btn_formulas_add.position = ( button_2_left, self.btn_nav_formula_next.get_bottom() + 20) self.btn_formulas_add.click_callback = self.btn_formulas_add_click self.container_action_buttons.append(self.btn_formulas_add) self.btn_formulas_del = ScreenButton("btn_formulas_del", "Del. Formula", 21, button_2_width) self.btn_formulas_del.set_colors(button_text_color, button_back_color) self.btn_formulas_del.position = ( button_2_right, self.btn_nav_formula_next.get_bottom() + 20) self.btn_formulas_del.click_callback = self.btn_formulas_del_click self.container_action_buttons.append(self.btn_formulas_del) self.btn_formula_update_tag = ScreenButton("btn_formula_update_tag", "Update Tag", 21, button_width) self.btn_formula_update_tag.set_colors(button_text_color, button_back_color) self.btn_formula_update_tag.position = ( button_left, self.btn_formulas_del.get_bottom() + 20) self.btn_formula_update_tag.click_callback = self.btn_formula_update_tag_click self.container_action_buttons.append(self.btn_formula_update_tag) self.lbl_formula_tag = ScreenLabel("lbl_formula_tag", "Tag: ?", 21, container_width - 10, 1) self.lbl_formula_tag.position = ( 5, self.btn_formula_update_tag.get_bottom() + 20) self.lbl_formula_tag.set_background(general_background) self.lbl_formula_tag.set_color(text_color) self.container_action_buttons.append(self.lbl_formula_tag) # =============================================== stats_background = (60, 50, 40) self.container_stats = ScreenContainer("container_stats", (container_width, 70), back_color=stats_background) self.container_stats.position = ( self.width - container_width - 10, self.container_action_buttons.get_bottom() + 5) self.elements.append(self.container_stats) self.lbl_cc_stats = ScreenLabel("lbl_cc_stats", "Connected Component Stats", 21, container_width - 10, 1) self.lbl_cc_stats.position = (5, 5) self.lbl_cc_stats.set_background(stats_background) self.lbl_cc_stats.set_color(text_color) self.container_stats.append(self.lbl_cc_stats) self.lbl_cc_raw = ScreenLabel("lbl_cc_raw", "Total Raw CC:\n" + str(self.cc_total), 21, button_2_width, 1) self.lbl_cc_raw.position = (button_2_left, self.lbl_cc_stats.get_bottom() + 10) self.lbl_cc_raw.set_background(stats_background) self.lbl_cc_raw.set_color(text_color) self.container_stats.append(self.lbl_cc_raw) self.lbl_cc_unique = ScreenLabel( "lbl_cc_unique", "Total Unique CC:\n" + str(len(self.unique_groups)), 21, button_2_width, 1) self.lbl_cc_unique.position = (button_2_right, self.lbl_cc_stats.get_bottom() + 10) self.lbl_cc_unique.set_background(stats_background) self.lbl_cc_unique.set_color(text_color) self.container_stats.append(self.lbl_cc_unique) #============================================================= # Panel with state buttons (Undo, Redo, Save) self.container_state_buttons = ScreenContainer( "container_state_buttons", (container_width, 200), general_background) self.container_state_buttons.position = ( self.container_view_buttons.get_left(), self.container_stats.get_bottom() + 10) self.elements.append(self.container_state_buttons) self.btn_undo = ScreenButton("btn_undo", "Undo", 21, button_width) self.btn_undo.set_colors(button_text_color, button_back_color) self.btn_undo.position = (button_left, 5) self.btn_undo.click_callback = self.btn_undo_click self.container_state_buttons.append(self.btn_undo) self.btn_redo = ScreenButton("btn_redo", "Redo", 21, button_width) self.btn_redo.set_colors(button_text_color, button_back_color) self.btn_redo.position = (button_left, self.btn_undo.get_bottom() + 10) self.btn_redo.click_callback = self.btn_redo_click self.container_state_buttons.append(self.btn_redo) self.btn_save = ScreenButton("btn_save", "Save", 21, button_width) self.btn_save.set_colors(button_text_color, button_back_color) self.btn_save.position = (button_left, self.btn_redo.get_bottom() + 10) self.btn_save.click_callback = self.btn_save_click self.container_state_buttons.append(self.btn_save) self.btn_exit = ScreenButton("btn_exit", "Exit", 21, button_width) self.btn_exit.set_colors(button_text_color, button_back_color) self.btn_exit.position = (button_left, self.btn_save.get_bottom() + 30) self.btn_exit.click_callback = self.btn_exit_click self.container_state_buttons.append(self.btn_exit) # ============================================================== image_width = self.width - self.container_nav_buttons.width - 30 image_height = self.height - container_top - 10 self.container_images = ScreenContainer("container_images", (image_width, image_height), back_color=(0, 0, 0)) self.container_images.position = (10, container_top) self.elements.append(self.container_images) # ... image objects ... tempo_blank = np.zeros((50, 50, 3), np.uint8) tempo_blank[:, :, :] = 255 self.img_main = ScreenImage("img_main", tempo_blank, 0, 0, True, cv2.INTER_NEAREST) self.img_main.position = (0, 0) self.img_main.mouse_button_down_callback = self.img_mouse_down self.container_images.append(self.img_main) # canvas used for annotations self.canvas_select = ScreenCanvas("canvas_select", 100, 100) self.canvas_select.position = (0, 0) self.canvas_select.locked = False # self.canvas_select.object_edited_callback = self.canvas_object_edited # self.canvas_select.object_selected_callback = self.canvas_selection_changed self.container_images.append(self.canvas_select) self.canvas_select.add_element("selection_rectangle", 10, 10, 40, 40) self.canvas_select.elements["selection_rectangle"].visible = False self.undo_stack = [] self.redo_stack = [] self.update_selected_formula(0) self.update_current_view(True)
def keyframes_unique_cc(keyframe_set, alignments, local_window, min_recall, min_precision, verbose=False): percentiles = [25, 50, 75, 100] total_raw_cc = 0 h, w, _ = keyframe_set[0].raw_image.shape all_sizes = [] size_percentiles = [] cc_groups = [] for keyframe in keyframe_set: if keyframe.binary_cc is None: keyframe.update_binary_cc() total_raw_cc += len(keyframe.binary_cc) local_groups = {} for cc in keyframe.binary_cc: all_sizes.append(cc.size) local_groups[cc.strID()] = None cc_groups.append(local_groups) # print(" Key-frame: " + str(keyframe.idx) + ", found: " + str(len(current_cc)) + " CCs") if verbose: print("\tRaw CC count: " + str(total_raw_cc)) # compute size statistics all_sizes = np.array(all_sizes) print("\t-> Percentiles:") for percent in percentiles: tempo_percentile = np.percentile(all_sizes, percent) size_percentiles.append(tempo_percentile) print("\t\t" + str(percent) + "%\t<\t" + str(tempo_percentile)) # create mapping for first frame (everything is unique) unique_ccs = [] active_ccs = [] for cc in keyframe_set[0].binary_cc: new_group = UniqueCCGroup(cc, 0) unique_ccs.append(new_group) cc_groups[0][cc.strID()] = new_group active_ccs.append(new_group) for kf_idx in range(1, len(keyframe_set)): keyframe = keyframe_set[kf_idx] not_yet_found = list(active_ccs) active_ccs = [] align = alignments[kf_idx - 1] # print((kf_idx, align)) # for each cc in the current key-frame .... for kf_cc in keyframe.binary_cc: # check if similar enough to elements in active cc found = False for nyf_idx, active_cc in enumerate(not_yet_found): if Evaluator.check_equivalent_cc(kf_cc, active_cc.cc_refs[-1], align, local_window, min_recall, min_precision): # mark this CC as still active ... active_ccs.append(active_cc) # add cc instance to the group active_cc.cc_refs.append(kf_cc) # create inverse link to group cc_groups[kf_idx][kf_cc.strID()] = active_cc # remove from the list of CCs that still need to be found del not_yet_found[nyf_idx] # mark as found ... found = True break if not found: new_group = UniqueCCGroup(kf_cc, kf_idx) # add the new CC to the list of unique unique_ccs.append(new_group) # add inverse reference cc_groups[kf_idx][kf_cc.strID()] = new_group # mark as active ... active_ccs.append(new_group) if verbose: print("\tUnique CC count: " + str(len(unique_ccs))) unique_sizes = np.array( [group.cc_refs[0].size for group in unique_ccs]) print("\t-> Percentiles:") for percent in percentiles: tempo_percentile = np.percentile(unique_sizes, percent) # size_percentiles.append(tempo_percentile) print("\t\t" + str(percent) + "%\t<\t" + str(tempo_percentile)) return unique_ccs, cc_groups
def btn_confirm_accept_click(self, button): if self.edition_mode == GTUniqueCCAnnotator.ModeMatch_RegionSelection: # compute potential matches ... self.greedy_matching_scores() # clear base image where matches will be shown ... gray_mask = self.base_matching.sum(axis=2) < 255 * 3 self.base_matching[gray_mask, 0] = 128 self.base_matching[gray_mask, 1] = 128 self.base_matching[gray_mask, 2] = 128 # move to the next stage .. self.set_editor_mode(GTUniqueCCAnnotator.ModeMatch_Matching) self.update_current_view(False) elif self.edition_mode == GTUniqueCCAnnotator.ModeMatch_Matching: # accept matches for precision, recall, prev_cc, curr_cc in self.matching_scores: if precision >= self.matching_min_precision and recall >= self.matching_min_recall: # merge to previous group ... prev_id = prev_cc.strID() prev_group = self.cc_group[self.selected_keyframe - 1][prev_id] curr_id = curr_cc.strID() curr_group = self.cc_group[self.selected_keyframe][curr_id] # for each member of the current group ... for kf_offset, cc in enumerate(curr_group.cc_refs): # make element point to previous group cc_id = cc.strID() #print(self.cc_group[self.selected_keyframe + kf_offset][cc_id].start_frame) self.cc_group[self.selected_keyframe + kf_offset][cc_id] = prev_group #print(self.cc_group[self.selected_keyframe + kf_offset][cc_id].start_frame) # add element in current group to previous group prev_group.cc_refs.append(cc) # remove group from list of unique groups self.unique_groups.remove(curr_group) # update count .... self.lbl_cc_unique.set_text("Total Unique CC:\n" + str(len(self.unique_groups))) print("PENDING UNDO/REDO") # update colored images ... self.update_colored_cache(self.selected_keyframe) self.set_editor_mode(GTUniqueCCAnnotator.ModeNavigate) self.update_current_view(False) elif self.edition_mode == GTUniqueCCAnnotator.ModeMatch_Remove: sel_rect = self.canvas_select.elements["selection_rectangle"] rect_x = int(round(sel_rect.x / self.view_scale)) rect_y = int(round(sel_rect.y / self.view_scale)) rect_w = int(round(sel_rect.w / self.view_scale)) rect_h = int(round(sel_rect.h / self.view_scale)) # identify CC's from current frame within selected region (containment) curr_kf = self.keyframe_annotations[self.selected_keyframe] curr_ccs = curr_kf.ccs_in_region(rect_x, rect_x + rect_w, rect_y, rect_y + rect_h) curr_ccs = {cc.strID(): cc for cc in curr_ccs} # only keep those that have been previously matched filtered_ccs = {} for curr_cc_str_id in curr_ccs: if self.cc_group[self.selected_keyframe][curr_cc_str_id].start_frame < self.selected_keyframe: # matched CC ... filtered_ccs[curr_cc_str_id] = curr_ccs[curr_cc_str_id] print("Total CC in region (C-KF): " + str(len(curr_ccs))) print("Total matches to remove (C-KF): " + str(len(filtered_ccs))) curr_ccs = filtered_ccs # Remove CC's from group (split) and add their own group for curr_cc_str_id in curr_ccs: # previous group prev_group = self.cc_group[self.selected_keyframe][curr_cc_str_id] # ask the group to split new_group = UniqueCCGroup.Split(prev_group, self.selected_keyframe) # link CCs on the new group to the new group for split_offset, split_cc in enumerate(new_group.cc_refs): split_cc_str_id = split_cc.strID() self.cc_group[self.selected_keyframe + split_offset][split_cc_str_id] = new_group # add new group to the list of unique CC self.unique_groups.append(new_group) print("Pending to be able to UNDO/REDO") # update colored images ... self.update_colored_cache(self.selected_keyframe) self.set_editor_mode(GTUniqueCCAnnotator.ModeNavigate) self.update_current_view(False)
def __init__(self, size, db_name, lecture_title, output_path): Screen.__init__(self, "Unique CC Ground Truth Annotation Interface", size) general_background = (100, 90, 80) text_color = (255, 255, 255) button_text_color = (35, 50, 20) button_back_color = (228, 228, 228) self.elements.back_color = general_background self.db_name = db_name self.lecture_title = lecture_title self.output_path = output_path export_filename = self.output_path + "/segments.xml" export_image_prefix = self.output_path + "/keyframes/" # load including segment information self.keyframe_annotations, self.segments = KeyFrameAnnotation.LoadExportedKeyframes(export_filename, export_image_prefix, True) if len(self.keyframe_annotations) > 0: print("Key-frames Loaded: " + str(len(self.keyframe_annotations))) else: raise Exception("Cannot start with 0 key-frames") portions_filename = self.output_path + "/portions.xml" portions_path = self.output_path + "/portions/" if os.path.exists(portions_filename): # Saved data detected, loading print("Previously saved portion data detected, loading") KeyFrameAnnotation.LoadKeyframesPortions(portions_filename, self.keyframe_annotations, portions_path) else: raise Exception("No saved portion data detected, cannot continue") print("Original Key-frames: " + str(len(self.keyframe_annotations))) print("Segments: " + str(len(self.segments))) self.keyframe_annotations = KeyFrameAnnotation.CombineKeyframesPerSegment(self.keyframe_annotations, self.segments, True) print("Key-frames after combination per segment" + str(len(self.keyframe_annotations))) # other CC/group elements self.unique_groups = None self.cc_group = None self.colored_cache = [] self.cc_total = 0 for kf_idx, keyframe in enumerate(self.keyframe_annotations): self.cc_total += len(keyframe.binary_cc) unique_cc_filename = self.output_path + "/unique_ccs.xml" if os.path.exists(unique_cc_filename): # Saved data detected, loading print("Previously saved unique CC data detected, loading") self.cc_group, self.unique_groups = UniqueCCGroup.GroupsFromXML(self.keyframe_annotations, unique_cc_filename) else: # no previous data, build default index (all CCs are unique) self.unique_groups = [] self.cc_group = [] for kf_idx, keyframe in enumerate(self.keyframe_annotations): self.cc_group.append({}) for cc in keyframe.binary_cc: new_group = UniqueCCGroup(cc, kf_idx) self.unique_groups.append(new_group) self.cc_group[kf_idx][cc.strID()] = new_group self.update_colored_cache(0) self.view_mode = GTUniqueCCAnnotator.ViewModeColored self.edition_mode = GTUniqueCCAnnotator.ModeNavigate self.view_scale = 1.0 self.selected_keyframe = 0 self.matching_delta_x = 0 self.matching_delta_y = 0 self.matching_scores = None self.matching_min_recall = 0.99 self.matching_min_precision = 0.99 self.base_matching = None # add elements.... container_top = 10 container_width = 330 button_2_width = 150 button_2_left = int(container_width * 0.25) - button_2_width / 2 button_2_right = int(container_width * 0.75) - button_2_width / 2 # Navigation panel to move accross frames self.container_nav_buttons = ScreenContainer("container_nav_buttons", (container_width, 70), back_color=general_background) self.container_nav_buttons.position = (self.width - self.container_nav_buttons.width - 10, container_top) self.elements.append(self.container_nav_buttons) self.lbl_nav_keyframe = ScreenLabel("lbl_nav_keyframe", "Key-Frame: 1 / " + str(len(self.keyframe_annotations)), 21, 290, 1) self.lbl_nav_keyframe.position = (5, 5) self.lbl_nav_keyframe.set_background(general_background) self.lbl_nav_keyframe.set_color(text_color) self.container_nav_buttons.append(self.lbl_nav_keyframe) time_str = TimeHelper.stampToStr(self.keyframe_annotations[self.selected_keyframe].time) self.lbl_nav_time = ScreenLabel("lbl_nav_time", time_str, 21, 290, 1) self.lbl_nav_time.position = (5, self.lbl_nav_keyframe.get_bottom() + 20) self.lbl_nav_time.set_background(general_background) self.lbl_nav_time.set_color(text_color) self.container_nav_buttons.append(self.lbl_nav_time) self.btn_nav_keyframe_prev = ScreenButton("btn_nav_keyframe_prev", "Prev", 21, 90) self.btn_nav_keyframe_prev.set_colors(button_text_color, button_back_color) self.btn_nav_keyframe_prev.position = (10, self.lbl_nav_keyframe.get_bottom() + 10) self.btn_nav_keyframe_prev.click_callback = self.btn_nav_keyframe_prev_click self.container_nav_buttons.append(self.btn_nav_keyframe_prev) self.btn_nav_keyframe_next = ScreenButton("btn_nav_keyframe_next", "Next", 21, 90) self.btn_nav_keyframe_next.set_colors(button_text_color, button_back_color) self.btn_nav_keyframe_next.position = (self.container_nav_buttons.width - self.btn_nav_keyframe_next.width - 10, self.lbl_nav_keyframe.get_bottom() + 10) self.btn_nav_keyframe_next.click_callback = self.btn_nav_keyframe_next_click self.container_nav_buttons.append(self.btn_nav_keyframe_next) # confirmation panel self.container_confirm_buttons = ScreenContainer("container_confirm_buttons", (container_width, 70), back_color=general_background) self.container_confirm_buttons.position = (self.width - self.container_confirm_buttons.width - 10, container_top) self.elements.append(self.container_confirm_buttons) self.container_confirm_buttons.visible = False self.lbl_confirm_message = ScreenLabel("lbl_confirm_message", "Confirmation message goes here?", 21, 290, 1) self.lbl_confirm_message.position = (5, 5) self.lbl_confirm_message.set_background(general_background) self.lbl_confirm_message.set_color(text_color) self.container_confirm_buttons.append(self.lbl_confirm_message) self.btn_confirm_cancel = ScreenButton("btn_confirm_cancel", "Cancel", 21, 130) self.btn_confirm_cancel.set_colors(button_text_color, button_back_color) self.btn_confirm_cancel.position = (10, self.lbl_nav_keyframe.get_bottom() + 10) self.btn_confirm_cancel.click_callback = self.btn_confirm_cancel_click self.container_confirm_buttons.append(self.btn_confirm_cancel) self.btn_confirm_accept = ScreenButton("btn_confirm_accept", "Accept", 21, 130) self.btn_confirm_accept.set_colors(button_text_color, button_back_color) self.btn_confirm_accept.position = (self.container_confirm_buttons.width - self.btn_confirm_accept.width - 10, self.lbl_confirm_message.get_bottom() + 10) self.btn_confirm_accept.click_callback = self.btn_confirm_accept_click self.container_confirm_buttons.append(self.btn_confirm_accept) # View panel with view control buttons self.container_view_buttons = ScreenContainer("container_view_buttons", (container_width, 165), back_color=general_background) self.container_view_buttons.position = (self.width - self.container_view_buttons.width - 10, self.container_nav_buttons.get_bottom() + 10) self.elements.append(self.container_view_buttons) button_width = 190 button_left = (self.container_view_buttons.width - button_width) / 2 # zoom .... self.lbl_zoom = ScreenLabel("lbl_zoom", "Zoom: 100%", 21, container_width - 10, 1) self.lbl_zoom.position = (5, 5) self.lbl_zoom.set_background(general_background) self.lbl_zoom.set_color(text_color) self.container_view_buttons.append(self.lbl_zoom) self.btn_zoom_reduce = ScreenButton("btn_zoom_reduce", "[ - ]", 21, 90) self.btn_zoom_reduce.set_colors(button_text_color, button_back_color) self.btn_zoom_reduce.position = (10, self.lbl_zoom.get_bottom() + 10) self.btn_zoom_reduce.click_callback = self.btn_zoom_reduce_click self.container_view_buttons.append(self.btn_zoom_reduce) self.btn_zoom_increase = ScreenButton("btn_zoom_increase", "[ + ]", 21, 90) self.btn_zoom_increase.set_colors(button_text_color, button_back_color) self.btn_zoom_increase.position = (self.container_view_buttons.width - self.btn_zoom_increase.width - 10, self.lbl_zoom.get_bottom() + 10) self.btn_zoom_increase.click_callback = self.btn_zoom_increase_click self.container_view_buttons.append(self.btn_zoom_increase) self.btn_zoom_zero = ScreenButton("btn_zoom_zero", "100%", 21, 90) self.btn_zoom_zero.set_colors(button_text_color, button_back_color) self.btn_zoom_zero.position = ((self.container_view_buttons.width - self.btn_zoom_zero.width) / 2, self.lbl_zoom.get_bottom() + 10) self.btn_zoom_zero.click_callback = self.btn_zoom_zero_click self.container_view_buttons.append(self.btn_zoom_zero) self.btn_view_raw = ScreenButton("btn_view_raw", "Raw View", 21, button_2_width) self.btn_view_raw.set_colors(button_text_color, button_back_color) self.btn_view_raw.position = (button_2_left, self.btn_zoom_zero.get_bottom() + 10) self.btn_view_raw.click_callback = self.btn_view_raw_click self.container_view_buttons.append(self.btn_view_raw) self.btn_view_gray = ScreenButton("btn_view_gray", "Grayscale View", 21, button_2_width) self.btn_view_gray.set_colors(button_text_color, button_back_color) self.btn_view_gray.position = (button_2_right, self.btn_zoom_zero.get_bottom() + 10) self.btn_view_gray.click_callback = self.btn_view_gray_click self.container_view_buttons.append(self.btn_view_gray) self.btn_view_binary = ScreenButton("btn_view_binary", "Binary View", 21, button_2_width) self.btn_view_binary.set_colors(button_text_color, button_back_color) self.btn_view_binary.position = (button_2_left, self.btn_view_gray.get_bottom() + 10) self.btn_view_binary.click_callback = self.btn_view_binary_click self.container_view_buttons.append(self.btn_view_binary) self.btn_view_colored = ScreenButton("btn_view_colored", "Colored View", 21, button_2_width) self.btn_view_colored.set_colors(button_text_color, button_back_color) self.btn_view_colored.position = (button_2_right, self.btn_view_gray.get_bottom() + 10) self.btn_view_colored.click_callback = self.btn_view_colored_click self.container_view_buttons.append(self.btn_view_colored) # Panel with action buttons (Add/Remove links) self.container_action_buttons = ScreenContainer("container_action_buttons", (container_width, 45), general_background) self.container_action_buttons.position = (self.container_view_buttons.get_left(), self.container_view_buttons.get_bottom() + 5) self.elements.append(self.container_action_buttons) self.btn_matches_add = ScreenButton("btn_matches_add", "Add Matches", 21, button_2_width) self.btn_matches_add.set_colors(button_text_color, button_back_color) self.btn_matches_add.position = (button_2_left, 5) self.btn_matches_add.click_callback = self.btn_matches_add_click self.container_action_buttons.append(self.btn_matches_add) self.btn_matches_del = ScreenButton("btn_matches_del", "Del. Matches", 21, button_2_width) self.btn_matches_del.set_colors(button_text_color, button_back_color) self.btn_matches_del.position = (button_2_right, 5) self.btn_matches_del.click_callback = self.btn_matches_del_click self.container_action_buttons.append(self.btn_matches_del) # =============================================== # Panel with matching parameters for step 1 (Matching Translation) self.container_matching_translation = ScreenContainer("container_matching_translation", (container_width, 150), general_background) self.container_matching_translation.position = (self.container_view_buttons.get_left(), self.container_view_buttons.get_bottom() + 5) self.elements.append(self.container_matching_translation) self.lbl_translation_title = ScreenLabel("lbl_translation_title", "Translation Parameters", 21, container_width - 10, 1) self.lbl_translation_title.position = (5, 5) self.lbl_translation_title.set_background(general_background) self.lbl_translation_title.set_color(text_color) self.container_matching_translation.append(self.lbl_translation_title) self.lbl_delta_x = ScreenLabel("lbl_delta_x", "Delta X: " + str(self.matching_delta_x), 21, container_width - 10, 1) self.lbl_delta_x.position = (5, self.lbl_translation_title.get_bottom() + 20) self.lbl_delta_x.set_background(general_background) self.lbl_delta_x.set_color(text_color) self.container_matching_translation.append(self.lbl_delta_x) max_delta = GTUniqueCCAnnotator.ParamsMaxTranslation self.scroll_delta_x = ScreenHorizontalScroll("scroll_delta_x", -max_delta, max_delta, 0, 1) self.scroll_delta_x.position = (5, self.lbl_delta_x.get_bottom() + 10) self.scroll_delta_x.width = container_width - 10 self.scroll_delta_x.scroll_callback = self.scroll_delta_x_change self.container_matching_translation.append(self.scroll_delta_x) self.lbl_delta_y = ScreenLabel("lbl_delta_y", "Delta Y: " + str(self.matching_delta_y), 21, container_width - 10, 1) self.lbl_delta_y.position = (5, self.scroll_delta_x.get_bottom() + 20) self.lbl_delta_y.set_background(general_background) self.lbl_delta_y.set_color(text_color) self.container_matching_translation.append(self.lbl_delta_y) self.scroll_delta_y = ScreenHorizontalScroll("scroll_delta_y", -max_delta, max_delta, 0, 1) self.scroll_delta_y.position = (5, self.lbl_delta_y.get_bottom() + 10) self.scroll_delta_y.width = container_width - 10 self.scroll_delta_y.scroll_callback = self.scroll_delta_y_change self.container_matching_translation.append(self.scroll_delta_y) self.container_matching_translation.visible = False # =============================================== # Panel with matching parameters for step 2 (Matching Strictness) self.container_matching_strictness = ScreenContainer("container_matching_strictness", (container_width, 150), general_background) self.container_matching_strictness.position = (self.container_view_buttons.get_left(), self.container_view_buttons.get_bottom() + 5) self.elements.append(self.container_matching_strictness) self.lbl_matching_title = ScreenLabel("lbl_matching_title", "Matching Parameters", 21, container_width - 10, 1) self.lbl_matching_title.position = (5, 5) self.lbl_matching_title.set_background(general_background) self.lbl_matching_title.set_color(text_color) self.container_matching_strictness.append(self.lbl_matching_title) str_recall = "Minimum Recall: " + str(int(self.matching_min_recall * 100)) self.lbl_min_recall = ScreenLabel("lbl_min_recall", str_recall, 21, container_width - 10, 1) self.lbl_min_recall.position = (5, self.lbl_matching_title.get_bottom() + 20) self.lbl_min_recall.set_background(general_background) self.lbl_min_recall.set_color(text_color) self.container_matching_strictness.append(self.lbl_min_recall) min_recall = GTUniqueCCAnnotator.ParamsMinRecall self.scroll_min_recall = ScreenHorizontalScroll("scroll_min_recall", min_recall, 100, 99, 1) self.scroll_min_recall.position = (5, self.lbl_min_recall.get_bottom() + 10) self.scroll_min_recall.width = container_width - 10 self.scroll_min_recall.scroll_callback = self.scroll_min_recall_change self.container_matching_strictness.append(self.scroll_min_recall) str_precision = "Minimum Precision: " + str(int(self.matching_min_precision * 100)) self.lbl_min_precision = ScreenLabel("lbl_min_precision", str_precision, 21, container_width - 10, 1) self.lbl_min_precision.position = (5, self.scroll_min_recall.get_bottom() + 20) self.lbl_min_precision.set_background(general_background) self.lbl_min_precision.set_color(text_color) self.container_matching_strictness.append(self.lbl_min_precision) min_precision = GTUniqueCCAnnotator.ParamsMinPrecision self.scroll_min_precision = ScreenHorizontalScroll("scroll_min_precision", min_precision, 100, 99, 1) self.scroll_min_precision.position = (5, self.lbl_min_precision.get_bottom() + 10) self.scroll_min_precision.width = container_width - 10 self.scroll_min_precision.scroll_callback = self.scroll_min_precision_change self.container_matching_strictness.append(self.scroll_min_precision) self.container_matching_strictness.visible = False # =============================================== stats_background = (60, 50, 40) self.container_stats = ScreenContainer("container_stats", (container_width, 70), back_color=stats_background) self.container_stats.position = (self.width - container_width - 10, self.container_action_buttons.get_bottom() + 5) self.elements.append(self.container_stats) self.lbl_cc_stats = ScreenLabel("lbl_cc_stats", "Connected Component Stats", 21, container_width - 10, 1) self.lbl_cc_stats.position = (5, 5) self.lbl_cc_stats.set_background(stats_background) self.lbl_cc_stats.set_color(text_color) self.container_stats.append(self.lbl_cc_stats) self.lbl_cc_raw = ScreenLabel("lbl_cc_raw", "Total Raw CC:\n" + str(self.cc_total), 21, button_2_width, 1) self.lbl_cc_raw.position = (button_2_left, self.lbl_cc_stats.get_bottom() + 10) self.lbl_cc_raw.set_background(stats_background) self.lbl_cc_raw.set_color(text_color) self.container_stats.append(self.lbl_cc_raw) self.lbl_cc_unique = ScreenLabel("lbl_cc_unique", "Total Unique CC:\n" + str(len(self.unique_groups)), 21, button_2_width, 1) self.lbl_cc_unique.position = (button_2_right, self.lbl_cc_stats.get_bottom() + 10) self.lbl_cc_unique.set_background(stats_background) self.lbl_cc_unique.set_color(text_color) self.container_stats.append(self.lbl_cc_unique) #============================================================= # Panel with state buttons (Undo, Redo, Save) self.container_state_buttons = ScreenContainer("container_state_buttons", (container_width, 200), general_background) self.container_state_buttons.position = ( self.container_view_buttons.get_left(), self.container_stats.get_bottom() + 10) self.elements.append(self.container_state_buttons) self.btn_undo = ScreenButton("btn_undo", "Undo", 21, button_width) self.btn_undo.set_colors(button_text_color, button_back_color) self.btn_undo.position = (button_left, 5) self.btn_undo.click_callback = self.btn_undo_click self.container_state_buttons.append(self.btn_undo) self.btn_redo = ScreenButton("btn_redo", "Redo", 21, button_width) self.btn_redo.set_colors(button_text_color, button_back_color) self.btn_redo.position = (button_left, self.btn_undo.get_bottom() + 10) self.btn_redo.click_callback = self.btn_redo_click self.container_state_buttons.append(self.btn_redo) self.btn_save = ScreenButton("btn_save", "Save", 21, button_width) self.btn_save.set_colors(button_text_color, button_back_color) self.btn_save.position = (button_left, self.btn_redo.get_bottom() + 10) self.btn_save.click_callback = self.btn_save_click self.container_state_buttons.append(self.btn_save) self.btn_exit = ScreenButton("btn_exit", "Exit", 21, button_width) self.btn_exit.set_colors(button_text_color, button_back_color) self.btn_exit.position = (button_left, self.btn_save.get_bottom() + 30) self.btn_exit.click_callback = self.btn_exit_click self.container_state_buttons.append(self.btn_exit) # print("MAKE CONTAINER STATS VISIBLE AGAIN!!!") # self.container_stats.visible = False # self.container_state_buttons.visible = False # ============================================================== image_width = self.width - self.container_nav_buttons.width - 30 image_height = self.height - container_top - 10 self.container_images = ScreenContainer("container_images", (image_width, image_height), back_color=(0, 0, 0)) self.container_images.position = (10, container_top) self.elements.append(self.container_images) # ... image objects ... tempo_blank = np.zeros((50, 50, 3), np.uint8) tempo_blank[:, :, :] = 255 self.img_main = ScreenImage("img_main", tempo_blank, 0, 0, True, cv2.INTER_NEAREST) self.img_main.position = (0, 0) #self.img_main.mouse_button_down_callback = self.img_mouse_down self.container_images.append(self.img_main) # canvas used for annotations self.canvas_select = ScreenCanvas("canvas_select", 100, 100) self.canvas_select.position = (0, 0) self.canvas_select.locked = False # self.canvas_select.object_edited_callback = self.canvas_object_edited # self.canvas_select.object_selected_callback = self.canvas_selection_changed self.container_images.append(self.canvas_select) self.canvas_select.add_element("selection_rectangle", 10, 10, 40, 40) self.canvas_select.elements["selection_rectangle"].visible = False self.undo_stack = [] self.redo_stack = [] self.update_current_view(True)