def load_correction_file(correction_file_path): droplet_corrections = {} correction_file = open(correction_file_path, "r") lines = correction_file.readlines() correction_file.close() correction_count = 0 for i, line in enumerate(lines): if line.strip() == "" or line[0] == "#": continue else: parts = line.split("#") matches = re.findall(r"(\d+)", parts[0]) if len(matches) == 2: droplet_corrections[int(matches[0])] = int(matches[1]) correction_count += 1 elif len(matches) == 1: droplet_corrections[int(matches[0])] = None correction_count += 1 else: printc( "\nOops! Correction file line {} isn't something we expected:\n {}\n" .format(i + 1, line), "red", ) return droplet_corrections, correction_count
def _print_status(self, calling_function): # Debug. printc( """ function = {} history_retrieval_point = {} _PROCESSED_HISTORY = {} in_history = {} """.format( calling_function, self.history_retrieval_point, self._PROCESSED_HISTORY, self.in_history, ), 'yellow', )
def _process(self, frame, index_frame_number): """""" droplet_data = self._video_master.frames[index_frame_number].droplets # print( # "frame: {}, {} droplets found".format(index_frame_number, len(droplet_data)) # ) # Debug self.video_total_unprocessed_droplet_count += len(droplet_data) # We want the grayscale frame with the border cleaned up, but # we don't want the droplets. thresholded_frame = threshold_and_find_droplets(frame, self.image_threshold, self.border_width, DROPLET_SCAN=False) # Introduce this frame. if self._VERBOSE: areas = [droplet_data[x].area for x in droplet_data] if len(areas) > 0: areas_string = "(" + " ".join([str(x) for x in areas]) + ")" else: areas_string = "" printc( "----- Frame {}: {} raw droplet{}, {} pixel{} {} ---------------" .format( self.index_frame_number + 1, len(droplet_data), ess(len(droplet_data)), sum(areas), ess(sum(areas)), areas_string, ), "purple", ) # # Tracker interlude # # Most of the shenanigans happen here. All the droplets go out, but # some don't come back. winnowed_droplets = self._droplet_tracker.update( new_droplet_dict=droplet_data, this_frame=self.index_frame_number) # # Beginning of pretty video frame. # # Convert frame back to color so we can write in color on it. self.processed_frame = cv2.cvtColor(thresholded_frame, cv2.COLOR_GRAY2RGB) self.frame_droplet_count = 0 self.frame_pixel_area = 0 frame_area_correction = 0 # # Highlight and label found droplets. # labels = Labeler( index_frame_number, frame_shape=self.frame_shape, VERBOSE=self._VERBOSE, DEBUG=self._DEBUG, ) for droplet_id in winnowed_droplets: new_droplet = False # Our flag for dealing with counts and areas later. # Check to see if this droplet was matched to a prior frame. if (droplet_id not in self._video_master.index_by_frame[ self.index_frame_number]): # We think the droplet is a match to a prior frame. :) new_droplet = True # "New droplet" as in "this droplet number isn't one we # expected in this frame." # The default droplet pixel area is the area of the most recent # droplet sighting. We might be able to do better here, for # instance getting the max area from the multiple sightings, # but for now let's be simple, and use the original area. # We might need to subtract out the current contour area # later: save the correction. frame_area_correction += self._video_master.index_by_droplet[ droplet_id].area # Get the data for this droplet. droplet = self._video_master.index_by_droplet[droplet_id] label = labels.add_label(droplet) if not new_droplet: self.frame_droplet_count += 1 if not self._reprocessing: self.video_total_droplet_count += 1 # If we need video, either for captured file or end-user display # while processing or to capture the top 10 frame images. # if not HIDE_VIDEO or CAPTURE_VIDEO or TOP_10: if not self._HIDE_DROPLET_HISTORY: # Draw outlines of any prior generations. if droplet.generations() >= 2: # Contour history to draw before the green box. for contour in droplet.contour_history(): self.processed_frame = cv2.drawContours( self.processed_frame, contour, -1, config.amber) # Draw red bounding box around this frame's contour. label.draw_contour_bounding_box(self.processed_frame, color=config.bright_red, thickness=1) # Mark the center with a single red pixel. # (This will never be seen, unless the frame is grabbed and # magnified. :) integer_droplet_center = tuple([int(n) for n in droplet.centroid]) cv2.line( self.processed_frame, integer_droplet_center, integer_droplet_center, config.bright_red, 1, ) # Getting the droplet area has already been done for us in the file scan. area = droplet.area # if new_droplet: self.frame_pixel_area += area self.video_total_pixel_area += area if self._csv_file: self._csv_file.update_csv_row( str(self.counting_frame_number), str(droplet.initial_id), [ droplet_id, area, "{:.2f}".format(droplet.centroid[0]), "{:.2f}".format(droplet.centroid[1]), ], ) # if self.index_frame_number >= 116: # Debug breakpoint catcher # debug_catcher = True # Draw all the labels. labels.draw(self.processed_frame) # Add some frame labeling. self.processed_frame = add_frame_header_text( self.processed_frame, get_filename_from_path(self.file_path), self.counting_frame_number, self.file_length_in_frames, self.frame_droplet_count, self.frame_pixel_area, self.video_total_droplet_count, self.video_total_unprocessed_droplet_count, self.video_total_pixel_area, self.image_threshold, self.history, self.similarity_threshold, self.distance_threshold, ) # Update and draw the droplet graph. if self._reprocessing: self._tiny_graph.reset_max_y( max(self._video_master.droplet_counts_by_frame)) else: self._tiny_graph.update( len(droplet_data), self.audio_data_by_frame[self.index_frame_number]) self._tiny_graph.canvas = self.processed_frame self.processed_frame = self._tiny_graph.draw_graph() # Composite annotations on to original video frame. self.processed_frame = cv2.add( add_alpha_channel(frame), add_alpha_channel(self.processed_frame, transparent_color=(0, 0, 0)), ) # Capture the output frame. # if self._CAPTURE_VIDEO: # # Not sure if this is needed... # self.processed_frame = remove_alpha_channel(self.processed_frame) # for _ in range(self._output_frames): # self.video_output.write(self.processed_frame.astype("uint8")) # cv2.imwrite( # '/Users/CS255/Desktop/git/python/fmva/test_output_3/test.jpg', # self.processed_frame, # ) # Put the finished frame back into the dispenser. self._frame_dispenser.processed_frame_return(self.processed_frame) if self._reprocessing: self._reprocessing = False return self.processed_frame
def update(self, new_droplet_dict=None, this_frame=None, BACK=False): """ Update the tracker. TODO Tracker.update() was written as a one-way data transformation. In TODO hindsight, it would be useful to back up through a video file and TODO recalculate droplet tracking going forward. This will require tracking TODO updates to save a prior frame's tracking data state. :param new_droplet_dict: dict of raw droplets found in current frame :param this_frame: current frame number, used in corrections :return: winnowed droplet dict, filtered for already found droplets, """ if len(new_droplet_dict) != 0: self._first_droplet = min(new_droplet_dict) else: self._first_droplet = 0 # if this_frame == 45: # debug_catcher = True # Placeholder. If we don't automatically identify any matched droplets in # this frame, but our correction file has an entry to be added in the frame, # there's an edge case that needs this to exist. matched_ids = OrderedDict() # 0. Candidate registry is populated with new droplets. if new_droplet_dict: self.droplet_candidate_registry = new_droplet_dict.copy() # 1. Bump all ages in history. (Newest goes to age 1.) for id in self._ageing: self._ageing[id] += 1 # 2. Remove history droplets with age greater than FRAMES_BEFORE_DEREGISTER for id in [ x for x in self._ageing if self._ageing[x] > self._FRAMES_BEFORE_DEREGISTER ]: self._deregister(id) # 3. Move current droplets to history. (Newest ones are now age 0.) for id in list(self.current_droplet_registry ): # Use list, so we don't mutate index. self._copy_to_history(id) self.current_droplet_registry.pop(id) # 4. Current should be empty: add new droplets to current. if len(self.current_droplet_registry) > 0: sys.exit("Oops. self.current_droplet_registry should be empty, " "but it still has {} droplets in it!").format( len(self.current_droplet_registry)) else: for droplet in self.droplet_candidate_registry: self._register(self.droplet_candidate_registry[droplet]) self.droplet_candidate_registry.clear() # 5. If we have droplets in this frame and history, compare distances. if (len(self.current_droplet_registry) > 0 and len(self.droplet_history_registry) > 0): # Without either current droplets or droplets in history, a droplet # comparison doesn't make sense! # Create an MxN array of distances between each known droplet and # each new droplet center. current_droplet_centroids = np.array( self._get_centroids(self.current_droplet_registry)) droplet_history_centroids = np.array( self._get_centroids(self.droplet_history_registry)) distances = distance.cdist(droplet_history_centroids, current_droplet_centroids, "euclidean") # Generate sorted list of rows, smallest distance first. distance_row_sort = distances.min(axis=1).argsort() # And the same for columns. distance_column_sort = distances.argmin(axis=1)[distance_row_sort] # Let's visualize distances. # This is now the default. I started with the thought that we might # need to have a quiet mode, but went the other way, not only chattering # in the output, but offering to save the colorful console output to # an html log file. distance_highlight_column = distance_column_sort[0] + 1 # (+1 because we're adding droplet numbers to the left side of # the table.) distance_highlight_row = distance_row_sort[0] if self._SHOW_DROPLET_TABLE and self._VERBOSE: printc("\nDistance", "bright red") print( self._print_distance_array( distances, self.droplet_history_registry.keys(), self.current_droplet_registry.keys(), highlight_column=distance_highlight_column, highlight_row=distance_highlight_row, color="red", )) # print("distance_row_sort: {}".format(distance_row_sort)) # Debug # print( # "self.droplet_history_registry.keys(): {}".format( # list(self.droplet_history_registry.keys()) # ) # ) # Debug # 6. Create shape comparison matrix for history vs current. current_droplet_contours = [] for id in self.current_droplet_registry: current_droplet_contours.append( self.current_droplet_registry[id].contour) droplet_history_contours = [] for id in self.droplet_history_registry: droplet_history_contours.append( self.droplet_history_registry[id].contour) shape_similarity = np.zeros( (len(droplet_history_contours), len(current_droplet_contours)), dtype=float, ) for row_index, history_contour in enumerate( droplet_history_contours): for col_index, current_contour in enumerate( current_droplet_contours): raw_similarity_score = cv2.matchShapes( history_contour, current_contour, cv2.CONTOURS_MATCH_I2, 0) # Biiiiig numbers. (Mostly not dealing well with 0 and inf.) I # should probably do a log transform on the raw hu moments first, # and do my own calcs, but matchShapes is convenient, and it'll be # in the right ballpark. transformed_score = abs( log_transform(raw_similarity_score)) if 308.0 < transformed_score < 308.5: # Edge case, close to 308.25 from float; too few pixels for # moments to work. transformed_score = 0.5 # SWAG that seems to mostly work. shape_similarity[row_index, col_index] = transformed_score # Use an array mask to filter out large numbers and (effectively) zeroes # and less. mshape_similarity = shape_similarity # mshape_similarity = ma.masked_outside(shape_similarity, 0.000001, 100.0) # We're going to use the distance sort to drive the decision order. # However, here the similarity matrix will highlight the smallest similarity # value, which may not match the distance table. shape_row_sort = mshape_similarity.min(axis=1).argsort() shape_column_sort = mshape_similarity.argmin( axis=1)[shape_row_sort] shape_highlight_column = shape_column_sort[0] + 1 shape_highlight_row = shape_row_sort[0] if self._SHOW_DROPLET_TABLE and self._VERBOSE: printc("Similarity", "bright blue") print( self._print_distance_array( mshape_similarity, self.droplet_history_registry.keys(), self.current_droplet_registry.keys(), highlight_column=shape_highlight_column, highlight_row=shape_highlight_row, color="bright blue", )) if self._SHOW_DROPLET_SUMMARY and self._VERBOSE: # Ditto on default. new_string = " ".join([ "{: >7}".format( list(self.current_droplet_registry.keys())[x]) for x in distance_column_sort ]) old_string = " ".join([ "{: >7}".format( list(self.droplet_history_registry.keys())[x]) for x in distance_row_sort ]) distance_string = " ".join([ "{: >7.2f}".format(distances[x, y]) for x, y in zip(distance_row_sort, distance_column_sort) ]) # similarity_string = " ".join( # [ # "{: >7.2f} ({},{})".format(mshape_similarity[x, y], x, y) # for x, y in zip(shape_row_sort, shape_column_sort) # ] # ) # Debug - x,y of smallest similarity number. similarity_string = " ".join([ "{: >7.2f}".format(mshape_similarity[x, y]) for x, y in zip(distance_row_sort, distance_column_sort) ]) similarity_string = re.sub(r"--", " --", similarity_string) print(" new {}".format(new_string)) print(" old {}".format(old_string)) print(" distance {}".format(distance_string)) print("similarity {}\n".format(similarity_string)) # 7. ...back to making decisions on which droplets might be # previously seen... # For the time being, it looks like multiplying distance by our # similarity number gives us a decent indicator of whether a droplet # could be a re-sighting of a prior droplet. Let's start with # (d * s) < 5 as starting point for our guesses. # Sets used to track if a row/column pair has been used. used_rows = set() used_columns = set() matched_ids = OrderedDict() combinations_to_try = zip(distance_row_sort, distance_column_sort) for row, column in combinations_to_try: if row in used_rows or column in used_columns: # Skip this combination, as this pair has been matched. # printc("{}, {}".format(row, column), 'bright cyan') # Debug. continue similarity_factor = mshape_similarity[row, column] confidence = distances[row, column] * similarity_factor # Communicate. if confidence > self._CONFIDENCE_THRESHOLD: # Not a winner. This droplet isn't one we've seen before. if self._VERBOSE: printc( "Confidence: - {:.2f} - Droplets {} and {} are {:.2f} pixels apart, similarity = {:.2f}" .format( confidence, list( self.droplet_history_registry.keys())[row], list(self.current_droplet_registry.keys()) [column], distances[row, column], mshape_similarity[row, column], ), "red", ) elif distances[row, column] > self._DISTANCE_THRESHOLD: # Not a winner. This droplet isn't one we've seen before. if self._VERBOSE: printc( "Droplets {} and {} are {:.2f} pixels apart, greater than threshold of {}" .format( list( self.droplet_history_registry.keys())[row], list(self.current_droplet_registry.keys()) [column], distances[row, column], self._DISTANCE_THRESHOLD, ), "red", ) else: if self._VERBOSE: printc( "Confidence: + {:.2f} - Droplets {} and {} are {:.2f} pixels apart, similarity = {:.2f}" .format( confidence, list( self.droplet_history_registry.keys())[row], list(self.current_droplet_registry.keys()) [column], distances[row, column], mshape_similarity[row, column], ), "green", ) # self.distance_research["accepted"][ # ( # list(self.droplet_history_registry.keys())[row], # list(self.current_droplet_registry.keys())[column], # ) # ] = distances[row, column] # Remember the matched pair, and we'll update our registries # when we're done.. original_droplet_id = list( self.droplet_history_registry.keys())[row] new_droplet_id = list( self.current_droplet_registry.keys())[column] matched_ids[new_droplet_id] = original_droplet_id # Add the info to our tracking sets, so we don't look at # these again. used_rows.add(row) used_columns.add(column) if self._VERBOSE: print() # Loop over all matched droplet pairs, make needed registry changes # for corrections. for new_droplet_id in matched_ids: original_droplet_id = matched_ids[new_droplet_id] # Injecting droplet corrections. # There are three cases we're interested in. # Two happen inside this loop, looking at the matches we've found: # # 1. Don't make a droplet connection at all, even though we have # a match. # 2. Make a droplet connection other than the one that matched. # # And the third case is new droplet assignments we didn't catch: # # 3. Make a droplet connection when one wasn't matched at all. if (new_droplet_id in self._droplet_corrections and self._droplet_corrections[new_droplet_id] is None): # Case 1 - just skip this droplet. if self._VERBOSE: printc( "Droplet correction: droplet {} will *not* be connected to droplet {}" .format(new_droplet_id, original_droplet_id), "red", ) # self.distance_research["corrected"][ # (original_droplet_id, new_droplet_id) # ] = 0 continue elif (new_droplet_id in self._droplet_corrections and original_droplet_id != self._droplet_corrections[new_droplet_id]): # Case 2 - substitute new linkage. if self._VERBOSE: printc( "Droplet correction: droplet {} will be connected to droplet {} instead of {}" .format( new_droplet_id, self._droplet_corrections[new_droplet_id], original_droplet_id, ), "red", ) original_droplet_id = self._droplet_corrections[ new_droplet_id] self._process_droplet_connection(new_droplet_id, original_droplet_id) # And Case 3: correction droplets captured in this frame but were # otherwise ignored. for new_droplet_id in self._droplet_corrections: if (new_droplet_id in self._droplet_master.index_by_frame[this_frame] and new_droplet_id not in matched_ids): if self._droplet_corrections[new_droplet_id] is None: # Oops. If we're trying to connect one of the droplets in this frame # to a droplet in a prior frame. If this droplet correction doesn't # specify a droplet to connect to, then it's an error. if self._VERBOSE: printc( "Droplet correction oops: droplet {} doesn't have a prior connection." .format(new_droplet_id), "red", ) continue else: if self._VERBOSE: printc( "Droplet correction: droplet {} will be connected to droplet {}." .format( new_droplet_id, self._droplet_corrections[new_droplet_id], ), "red", ) self._process_droplet_connection( new_droplet_id, self._droplet_corrections[new_droplet_id]) return self.current_droplet_registry
def draw(self, video_frame): # Each droplet has four possible label positions, 0-3, starting in with the # upper left quadrant, and continuing clockwise. We'll need a data structure # for the label quadrants, to indicate if a quadrant cannot be considered or # is available for placement consideration. Possible quadrant states are: # # - available for use # - unavailable (for instance if it's too close to the edge of the frame) # - used for the droplet's label # - temporarily used in a fitting step, but can be reset back to unused # # 0. Check all labels to make sure none are being lost at frame edges. # Mark label rects within a pixel margin (10px?) of a screen edge # as unusable. # # Check for label too close to the edge of the video frame. # self._check_for_edge_closeness(video_frame) # # Find label collisions. # if len(self.labels) > 1: # If only one label in this frame, we've already checked proximity to the # frame edge, so skip. label_ids = sorted(list(self.labels)) # This visualizes all the potential collisions between droplet labels. # Creates an array of all combinations of pairs droplets in this frame # and all possible combinations of the four label areas in a two-droplet # combination. It populates the array with the pixel area of any overlap # of each label area between each specific pair of droplets in the frame. # All droplets in pairs, without replacement. This is one triangle in a # combination matrix, without the identity diagonal. id_combinations = list(combinations(label_ids, 2)) # All label corner area possibilities, with replacement and identities, as # we're comparing any possible combination of two independent sets of four. corner_combinations = list(product([0, 1, 2, 3], [0, 1, 2, 3])) label_overlaps = np.zeros( (len(id_combinations), len(corner_combinations)), dtype=np.int16) # Finds the area for each combination using the _and_ method # from Rectangle. for row_index, id_tuple in enumerate(id_combinations): for column_index, corner_tuple in enumerate( corner_combinations): label_overlaps[row_index, column_index] = ( self.labels[id_tuple[0]].text_bounding_box[ corner_tuple[0]] & self.labels[id_tuple[1]].text_bounding_box[ corner_tuple[1]]).area # Pretty-printing the resulting matrix. if self._DEBUG: printc( "\nLabel overlaps for frame {}\n".format( str(self.frame + 1)), "bright yellow", ) header_string = " {}".format(" ".join([ " {}/{}".format(*corner_combinations[x]) for x in range(len(corner_combinations)) ])) printc(header_string, "red") for row_index, id_tuple in enumerate(id_combinations): if self._DEBUG: printc("{:>4} {:>4}: ".format(id_tuple[0], id_tuple[1]), "red", end="") area_list = [ "{:>4}".format(label_overlaps[row_index][x]) for x in range(len(label_overlaps[row_index])) ] # TODO Some of this code, to identify and fix total overlaps between # TODO two droplets, might get moved to the "fix" section instead of here # TODO in "find." # The identity overlaps, ie area 1 with 1, 2 with 2, etc., happen when # a droplet is pretty much superimposed on another. Those combinations # are in spots 0, 5, 10 and 15 in our sorted list of keys. This uses a # boolean and on a byte string representing all possibilities that # overlap to find that condition, and sets the allowed label positions # to be diagonally opposite one another for the two droplets. (And it # colors those numbers blue in the table to make them easy to see.) bit_status_string = andbytes( b"1000010000100001", b"".join([ b"0" if label_overlaps[row_index][x] == 0 else b"1" for x in range(len(area_list)) ]), ) if bit_status_string == b"1000010000100001": area_list_string = " ".join([ start_color("bright blue") + area_list[x] + stop_color() if b"1000010000100001"[x] == 49 # integer byte value else area_list[x] for x in range(len(bit_status_string)) ]) area_list_string = (area_list_string + start_color("bright red") + " (complete overlap)" + stop_color()) # These two labels have overlaps in all four of their corner areas. # This gets the corners for the destination label of the vector, # tuple[1]. # corners = self._pick_corners( # vector( # self.labels[id_tuple[0]].center, # self.labels[id_tuple[1]].center, # ) # ) # We're setting the corner or corners to avoid to False, so # we'll get the reversed list. # TODO Ignoring this for now, until we see if the distance # TODO approach works to avoid collisions. # for corner in self._reverse_corners(corners): # self.labels[id_tuple[0]].corner_status[corner] = False # # And for the other label, avoid the original list. # for corner in corners: # self.labels[id_tuple[1]].corner_status[corner] = False # # print("Overlapping labels: {} and {}.".format(*id_tuple)) # Debug else: area_list_string = " ".join(area_list) # print(bit_status_string) if self._DEBUG: print(area_list_string) if self._DEBUG: print() # # Fix label overlaps. # # if label_overlaps.sum() != 0: # There are overlaps. if self.frame == 2: debug_Catcher = True # Debug - draw a red line connecting droplets if self._DEBUG: self._connect_dots(video_frame) # Pick a corner for the label, hopelfully one that won't # interfere with other droplets. self._choose_label_corners() if self._DEBUG: print() # # Make changes to and draw labels. # for label in self.labels: # if self.frame == 106: # debug_catcher = True # Just to make it easier to type and read... this_label = self.labels[label] # If this label doesn't have a corner defined from any previous operations. if not this_label.corner_used: # Assign label corner, starting with 0, checking for any # marked as False by frame edge test or collision code, etc. for corner in this_label.corner_status: if this_label.corner_status[corner] is not False: this_label.corner_used = corner break else: # Uh oh. No available corners. Force a corner # and complain. this_label.corner_used = 2 # printc("\n!!!\n!!! Label {} has no available corner!\n!!!" # .format(this_label.id), 'bright red') # Debug # For debugging and manual experimentation. if self._DEBUG: this_label.draw_all_corner_boxes(video_frame) # video_frame = this_label.draw_label(video_frame, 0) # video_frame = this_label.draw_label(video_frame, 1) # video_frame = this_label.draw_label(video_frame, 2) # video_frame = this_label.draw_label(video_frame, 3) video_frame = this_label.draw_label(video_frame, this_label.corner_used) return video_frame