def load_correction_file(correction_file_path):
droplet_corrections = {}
correction_file = open(correction_file_path, "r")
lines = correction_file.readlines()
correction_file.close()
correction_count = 0
for i, line in enumerate(lines):
if line.strip() == "" or line[0] == "#":
continue
else:
parts = line.split("#")
matches = re.findall(r"(\d+)", parts[0])
if len(matches) == 2:
droplet_corrections[int(matches[0])] = int(matches[1])
correction_count += 1
elif len(matches) == 1:
droplet_corrections[int(matches[0])] = None
correction_count += 1
else:
printc(
"\nOops! Correction file line {} isn't something we expected:\n {}\n"
.format(i + 1, line),
"red",
)
return droplet_corrections, correction_count
def _print_status(self, calling_function):
# Debug.
printc(
"""
function = {}
history_retrieval_point = {}
_PROCESSED_HISTORY = {}
in_history = {}
""".format(
calling_function,
self.history_retrieval_point,
self._PROCESSED_HISTORY,
self.in_history,
),
'yellow',
)
def _process(self, frame, index_frame_number):
""""""
droplet_data = self._video_master.frames[index_frame_number].droplets
# print(
# "frame: {}, {} droplets found".format(index_frame_number, len(droplet_data))
# ) # Debug
self.video_total_unprocessed_droplet_count += len(droplet_data)
# We want the grayscale frame with the border cleaned up, but
# we don't want the droplets.
thresholded_frame = threshold_and_find_droplets(frame,
self.image_threshold,
self.border_width,
DROPLET_SCAN=False)
# Introduce this frame.
if self._VERBOSE:
areas = [droplet_data[x].area for x in droplet_data]
if len(areas) > 0:
areas_string = "(" + " ".join([str(x) for x in areas]) + ")"
else:
areas_string = ""
printc(
"----- Frame {}: {} raw droplet{}, {} pixel{} {} ---------------"
.format(
self.index_frame_number + 1,
len(droplet_data),
ess(len(droplet_data)),
sum(areas),
ess(sum(areas)),
areas_string,
),
"purple",
)
#
# Tracker interlude
#
# Most of the shenanigans happen here. All the droplets go out, but
# some don't come back.
winnowed_droplets = self._droplet_tracker.update(
new_droplet_dict=droplet_data, this_frame=self.index_frame_number)
#
# Beginning of pretty video frame.
#
# Convert frame back to color so we can write in color on it.
self.processed_frame = cv2.cvtColor(thresholded_frame,
cv2.COLOR_GRAY2RGB)
self.frame_droplet_count = 0
self.frame_pixel_area = 0
frame_area_correction = 0
#
# Highlight and label found droplets.
#
labels = Labeler(
index_frame_number,
frame_shape=self.frame_shape,
VERBOSE=self._VERBOSE,
DEBUG=self._DEBUG,
)
for droplet_id in winnowed_droplets:
new_droplet = False # Our flag for dealing with counts and areas later.
# Check to see if this droplet was matched to a prior frame.
if (droplet_id not in self._video_master.index_by_frame[
self.index_frame_number]):
# We think the droplet is a match to a prior frame. :)
new_droplet = True
# "New droplet" as in "this droplet number isn't one we
# expected in this frame."
# The default droplet pixel area is the area of the most recent
# droplet sighting. We might be able to do better here, for
# instance getting the max area from the multiple sightings,
# but for now let's be simple, and use the original area.
# We might need to subtract out the current contour area
# later: save the correction.
frame_area_correction += self._video_master.index_by_droplet[
droplet_id].area
# Get the data for this droplet.
droplet = self._video_master.index_by_droplet[droplet_id]
label = labels.add_label(droplet)
if not new_droplet:
self.frame_droplet_count += 1
if not self._reprocessing:
self.video_total_droplet_count += 1
# If we need video, either for captured file or end-user display
# while processing or to capture the top 10 frame images.
# if not HIDE_VIDEO or CAPTURE_VIDEO or TOP_10:
if not self._HIDE_DROPLET_HISTORY:
# Draw outlines of any prior generations.
if droplet.generations() >= 2:
# Contour history to draw before the green box.
for contour in droplet.contour_history():
self.processed_frame = cv2.drawContours(
self.processed_frame, contour, -1, config.amber)
# Draw red bounding box around this frame's contour.
label.draw_contour_bounding_box(self.processed_frame,
color=config.bright_red,
thickness=1)
# Mark the center with a single red pixel.
# (This will never be seen, unless the frame is grabbed and
# magnified. :)
integer_droplet_center = tuple([int(n) for n in droplet.centroid])
cv2.line(
self.processed_frame,
integer_droplet_center,
integer_droplet_center,
config.bright_red,
1,
)
# Getting the droplet area has already been done for us in the file scan.
area = droplet.area
# if new_droplet:
self.frame_pixel_area += area
self.video_total_pixel_area += area
if self._csv_file:
self._csv_file.update_csv_row(
str(self.counting_frame_number),
str(droplet.initial_id),
[
droplet_id,
area,
"{:.2f}".format(droplet.centroid[0]),
"{:.2f}".format(droplet.centroid[1]),
],
)
# if self.index_frame_number >= 116: # Debug breakpoint catcher
# debug_catcher = True
# Draw all the labels.
labels.draw(self.processed_frame)
# Add some frame labeling.
self.processed_frame = add_frame_header_text(
self.processed_frame,
get_filename_from_path(self.file_path),
self.counting_frame_number,
self.file_length_in_frames,
self.frame_droplet_count,
self.frame_pixel_area,
self.video_total_droplet_count,
self.video_total_unprocessed_droplet_count,
self.video_total_pixel_area,
self.image_threshold,
self.history,
self.similarity_threshold,
self.distance_threshold,
)
# Update and draw the droplet graph.
if self._reprocessing:
self._tiny_graph.reset_max_y(
max(self._video_master.droplet_counts_by_frame))
else:
self._tiny_graph.update(
len(droplet_data),
self.audio_data_by_frame[self.index_frame_number])
self._tiny_graph.canvas = self.processed_frame
self.processed_frame = self._tiny_graph.draw_graph()
# Composite annotations on to original video frame.
self.processed_frame = cv2.add(
add_alpha_channel(frame),
add_alpha_channel(self.processed_frame,
transparent_color=(0, 0, 0)),
)
# Capture the output frame.
# if self._CAPTURE_VIDEO:
# # Not sure if this is needed...
# self.processed_frame = remove_alpha_channel(self.processed_frame)
# for _ in range(self._output_frames):
# self.video_output.write(self.processed_frame.astype("uint8"))
# cv2.imwrite(
# '/Users/CS255/Desktop/git/python/fmva/test_output_3/test.jpg',
# self.processed_frame,
# )
# Put the finished frame back into the dispenser.
self._frame_dispenser.processed_frame_return(self.processed_frame)
if self._reprocessing:
self._reprocessing = False
return self.processed_frame
def update(self, new_droplet_dict=None, this_frame=None, BACK=False):
"""
Update the tracker.
TODO Tracker.update() was written as a one-way data transformation. In
TODO hindsight, it would be useful to back up through a video file and
TODO recalculate droplet tracking going forward. This will require tracking
TODO updates to save a prior frame's tracking data state.
:param new_droplet_dict: dict of raw droplets found in current frame
:param this_frame: current frame number, used in corrections
:return: winnowed droplet dict, filtered for already found droplets,
"""
if len(new_droplet_dict) != 0:
self._first_droplet = min(new_droplet_dict)
else:
self._first_droplet = 0
# if this_frame == 45:
# debug_catcher = True
# Placeholder. If we don't automatically identify any matched droplets in
# this frame, but our correction file has an entry to be added in the frame,
# there's an edge case that needs this to exist.
matched_ids = OrderedDict()
# 0. Candidate registry is populated with new droplets.
if new_droplet_dict:
self.droplet_candidate_registry = new_droplet_dict.copy()
# 1. Bump all ages in history. (Newest goes to age 1.)
for id in self._ageing:
self._ageing[id] += 1
# 2. Remove history droplets with age greater than FRAMES_BEFORE_DEREGISTER
for id in [
x for x in self._ageing
if self._ageing[x] > self._FRAMES_BEFORE_DEREGISTER
]:
self._deregister(id)
# 3. Move current droplets to history. (Newest ones are now age 0.)
for id in list(self.current_droplet_registry
): # Use list, so we don't mutate index.
self._copy_to_history(id)
self.current_droplet_registry.pop(id)
# 4. Current should be empty: add new droplets to current.
if len(self.current_droplet_registry) > 0:
sys.exit("Oops. self.current_droplet_registry should be empty, "
"but it still has {} droplets in it!").format(
len(self.current_droplet_registry))
else:
for droplet in self.droplet_candidate_registry:
self._register(self.droplet_candidate_registry[droplet])
self.droplet_candidate_registry.clear()
# 5. If we have droplets in this frame and history, compare distances.
if (len(self.current_droplet_registry) > 0
and len(self.droplet_history_registry) > 0):
# Without either current droplets or droplets in history, a droplet
# comparison doesn't make sense!
# Create an MxN array of distances between each known droplet and
# each new droplet center.
current_droplet_centroids = np.array(
self._get_centroids(self.current_droplet_registry))
droplet_history_centroids = np.array(
self._get_centroids(self.droplet_history_registry))
distances = distance.cdist(droplet_history_centroids,
current_droplet_centroids, "euclidean")
# Generate sorted list of rows, smallest distance first.
distance_row_sort = distances.min(axis=1).argsort()
# And the same for columns.
distance_column_sort = distances.argmin(axis=1)[distance_row_sort]
# Let's visualize distances.
# This is now the default. I started with the thought that we might
# need to have a quiet mode, but went the other way, not only chattering
# in the output, but offering to save the colorful console output to
# an html log file.
distance_highlight_column = distance_column_sort[0] + 1
# (+1 because we're adding droplet numbers to the left side of
# the table.)
distance_highlight_row = distance_row_sort[0]
if self._SHOW_DROPLET_TABLE and self._VERBOSE:
printc("\nDistance", "bright red")
print(
self._print_distance_array(
distances,
self.droplet_history_registry.keys(),
self.current_droplet_registry.keys(),
highlight_column=distance_highlight_column,
highlight_row=distance_highlight_row,
color="red",
))
# print("distance_row_sort: {}".format(distance_row_sort)) # Debug
# print(
# "self.droplet_history_registry.keys(): {}".format(
# list(self.droplet_history_registry.keys())
# )
# ) # Debug
# 6. Create shape comparison matrix for history vs current.
current_droplet_contours = []
for id in self.current_droplet_registry:
current_droplet_contours.append(
self.current_droplet_registry[id].contour)
droplet_history_contours = []
for id in self.droplet_history_registry:
droplet_history_contours.append(
self.droplet_history_registry[id].contour)
shape_similarity = np.zeros(
(len(droplet_history_contours), len(current_droplet_contours)),
dtype=float,
)
for row_index, history_contour in enumerate(
droplet_history_contours):
for col_index, current_contour in enumerate(
current_droplet_contours):
raw_similarity_score = cv2.matchShapes(
history_contour, current_contour,
cv2.CONTOURS_MATCH_I2, 0)
# Biiiiig numbers. (Mostly not dealing well with 0 and inf.) I
# should probably do a log transform on the raw hu moments first,
# and do my own calcs, but matchShapes is convenient, and it'll be
# in the right ballpark.
transformed_score = abs(
log_transform(raw_similarity_score))
if 308.0 < transformed_score < 308.5:
# Edge case, close to 308.25 from float; too few pixels for
# moments to work.
transformed_score = 0.5 # SWAG that seems to mostly work.
shape_similarity[row_index, col_index] = transformed_score
# Use an array mask to filter out large numbers and (effectively) zeroes
# and less.
mshape_similarity = shape_similarity
# mshape_similarity = ma.masked_outside(shape_similarity, 0.000001, 100.0)
# We're going to use the distance sort to drive the decision order.
# However, here the similarity matrix will highlight the smallest similarity
# value, which may not match the distance table.
shape_row_sort = mshape_similarity.min(axis=1).argsort()
shape_column_sort = mshape_similarity.argmin(
axis=1)[shape_row_sort]
shape_highlight_column = shape_column_sort[0] + 1
shape_highlight_row = shape_row_sort[0]
if self._SHOW_DROPLET_TABLE and self._VERBOSE:
printc("Similarity", "bright blue")
print(
self._print_distance_array(
mshape_similarity,
self.droplet_history_registry.keys(),
self.current_droplet_registry.keys(),
highlight_column=shape_highlight_column,
highlight_row=shape_highlight_row,
color="bright blue",
))
if self._SHOW_DROPLET_SUMMARY and self._VERBOSE:
# Ditto on default.
new_string = " ".join([
"{: >7}".format(
list(self.current_droplet_registry.keys())[x])
for x in distance_column_sort
])
old_string = " ".join([
"{: >7}".format(
list(self.droplet_history_registry.keys())[x])
for x in distance_row_sort
])
distance_string = " ".join([
"{: >7.2f}".format(distances[x, y])
for x, y in zip(distance_row_sort, distance_column_sort)
])
# similarity_string = " ".join(
# [
# "{: >7.2f} ({},{})".format(mshape_similarity[x, y], x, y)
# for x, y in zip(shape_row_sort, shape_column_sort)
# ]
# ) # Debug - x,y of smallest similarity number.
similarity_string = " ".join([
"{: >7.2f}".format(mshape_similarity[x, y])
for x, y in zip(distance_row_sort, distance_column_sort)
])
similarity_string = re.sub(r"--", " --", similarity_string)
print(" new {}".format(new_string))
print(" old {}".format(old_string))
print(" distance {}".format(distance_string))
print("similarity {}\n".format(similarity_string))
# 7. ...back to making decisions on which droplets might be
# previously seen...
# For the time being, it looks like multiplying distance by our
# similarity number gives us a decent indicator of whether a droplet
# could be a re-sighting of a prior droplet. Let's start with
# (d * s) < 5 as starting point for our guesses.
# Sets used to track if a row/column pair has been used.
used_rows = set()
used_columns = set()
matched_ids = OrderedDict()
combinations_to_try = zip(distance_row_sort, distance_column_sort)
for row, column in combinations_to_try:
if row in used_rows or column in used_columns:
# Skip this combination, as this pair has been matched.
# printc("{}, {}".format(row, column), 'bright cyan') # Debug.
continue
similarity_factor = mshape_similarity[row, column]
confidence = distances[row, column] * similarity_factor
# Communicate.
if confidence > self._CONFIDENCE_THRESHOLD:
# Not a winner. This droplet isn't one we've seen before.
if self._VERBOSE:
printc(
"Confidence: - {:.2f} - Droplets {} and {} are {:.2f} pixels apart, similarity = {:.2f}"
.format(
confidence,
list(
self.droplet_history_registry.keys())[row],
list(self.current_droplet_registry.keys())
[column],
distances[row, column],
mshape_similarity[row, column],
),
"red",
)
elif distances[row, column] > self._DISTANCE_THRESHOLD:
# Not a winner. This droplet isn't one we've seen before.
if self._VERBOSE:
printc(
"Droplets {} and {} are {:.2f} pixels apart, greater than threshold of {}"
.format(
list(
self.droplet_history_registry.keys())[row],
list(self.current_droplet_registry.keys())
[column],
distances[row, column],
self._DISTANCE_THRESHOLD,
),
"red",
)
else:
if self._VERBOSE:
printc(
"Confidence: + {:.2f} - Droplets {} and {} are {:.2f} pixels apart, similarity = {:.2f}"
.format(
confidence,
list(
self.droplet_history_registry.keys())[row],
list(self.current_droplet_registry.keys())
[column],
distances[row, column],
mshape_similarity[row, column],
),
"green",
)
# self.distance_research["accepted"][
# (
# list(self.droplet_history_registry.keys())[row],
# list(self.current_droplet_registry.keys())[column],
# )
# ] = distances[row, column]
# Remember the matched pair, and we'll update our registries
# when we're done..
original_droplet_id = list(
self.droplet_history_registry.keys())[row]
new_droplet_id = list(
self.current_droplet_registry.keys())[column]
matched_ids[new_droplet_id] = original_droplet_id
# Add the info to our tracking sets, so we don't look at
# these again.
used_rows.add(row)
used_columns.add(column)
if self._VERBOSE:
print()
# Loop over all matched droplet pairs, make needed registry changes
# for corrections.
for new_droplet_id in matched_ids:
original_droplet_id = matched_ids[new_droplet_id]
# Injecting droplet corrections.
# There are three cases we're interested in.
# Two happen inside this loop, looking at the matches we've found:
#
# 1. Don't make a droplet connection at all, even though we have
# a match.
# 2. Make a droplet connection other than the one that matched.
#
# And the third case is new droplet assignments we didn't catch:
#
# 3. Make a droplet connection when one wasn't matched at all.
if (new_droplet_id in self._droplet_corrections
and self._droplet_corrections[new_droplet_id] is None):
# Case 1 - just skip this droplet.
if self._VERBOSE:
printc(
"Droplet correction: droplet {} will *not* be connected to droplet {}"
.format(new_droplet_id, original_droplet_id),
"red",
)
# self.distance_research["corrected"][
# (original_droplet_id, new_droplet_id)
# ] = 0
continue
elif (new_droplet_id in self._droplet_corrections
and original_droplet_id !=
self._droplet_corrections[new_droplet_id]):
# Case 2 - substitute new linkage.
if self._VERBOSE:
printc(
"Droplet correction: droplet {} will be connected to droplet {} instead of {}"
.format(
new_droplet_id,
self._droplet_corrections[new_droplet_id],
original_droplet_id,
),
"red",
)
original_droplet_id = self._droplet_corrections[
new_droplet_id]
self._process_droplet_connection(new_droplet_id,
original_droplet_id)
# And Case 3: correction droplets captured in this frame but were
# otherwise ignored.
for new_droplet_id in self._droplet_corrections:
if (new_droplet_id
in self._droplet_master.index_by_frame[this_frame]
and new_droplet_id not in matched_ids):
if self._droplet_corrections[new_droplet_id] is None:
# Oops. If we're trying to connect one of the droplets in this frame
# to a droplet in a prior frame. If this droplet correction doesn't
# specify a droplet to connect to, then it's an error.
if self._VERBOSE:
printc(
"Droplet correction oops: droplet {} doesn't have a prior connection."
.format(new_droplet_id),
"red",
)
continue
else:
if self._VERBOSE:
printc(
"Droplet correction: droplet {} will be connected to droplet {}."
.format(
new_droplet_id,
self._droplet_corrections[new_droplet_id],
),
"red",
)
self._process_droplet_connection(
new_droplet_id, self._droplet_corrections[new_droplet_id])
return self.current_droplet_registry
def draw(self, video_frame):
# Each droplet has four possible label positions, 0-3, starting in with the
# upper left quadrant, and continuing clockwise. We'll need a data structure
# for the label quadrants, to indicate if a quadrant cannot be considered or
# is available for placement consideration. Possible quadrant states are:
#
# - available for use
# - unavailable (for instance if it's too close to the edge of the frame)
# - used for the droplet's label
# - temporarily used in a fitting step, but can be reset back to unused
#
# 0. Check all labels to make sure none are being lost at frame edges.
# Mark label rects within a pixel margin (10px?) of a screen edge
# as unusable.
#
# Check for label too close to the edge of the video frame.
#
self._check_for_edge_closeness(video_frame)
#
# Find label collisions.
#
if len(self.labels) > 1:
# If only one label in this frame, we've already checked proximity to the
# frame edge, so skip.
label_ids = sorted(list(self.labels))
# This visualizes all the potential collisions between droplet labels.
# Creates an array of all combinations of pairs droplets in this frame
# and all possible combinations of the four label areas in a two-droplet
# combination. It populates the array with the pixel area of any overlap
# of each label area between each specific pair of droplets in the frame.
# All droplets in pairs, without replacement. This is one triangle in a
# combination matrix, without the identity diagonal.
id_combinations = list(combinations(label_ids, 2))
# All label corner area possibilities, with replacement and identities, as
# we're comparing any possible combination of two independent sets of four.
corner_combinations = list(product([0, 1, 2, 3], [0, 1, 2, 3]))
label_overlaps = np.zeros(
(len(id_combinations), len(corner_combinations)),
dtype=np.int16)
# Finds the area for each combination using the _and_ method
# from Rectangle.
for row_index, id_tuple in enumerate(id_combinations):
for column_index, corner_tuple in enumerate(
corner_combinations):
label_overlaps[row_index, column_index] = (
self.labels[id_tuple[0]].text_bounding_box[
corner_tuple[0]]
& self.labels[id_tuple[1]].text_bounding_box[
corner_tuple[1]]).area
# Pretty-printing the resulting matrix.
if self._DEBUG:
printc(
"\nLabel overlaps for frame {}\n".format(
str(self.frame + 1)),
"bright yellow",
)
header_string = " {}".format(" ".join([
" {}/{}".format(*corner_combinations[x])
for x in range(len(corner_combinations))
]))
printc(header_string, "red")
for row_index, id_tuple in enumerate(id_combinations):
if self._DEBUG:
printc("{:>4} {:>4}: ".format(id_tuple[0], id_tuple[1]),
"red",
end="")
area_list = [
"{:>4}".format(label_overlaps[row_index][x])
for x in range(len(label_overlaps[row_index]))
]
# TODO Some of this code, to identify and fix total overlaps between
# TODO two droplets, might get moved to the "fix" section instead of here
# TODO in "find."
# The identity overlaps, ie area 1 with 1, 2 with 2, etc., happen when
# a droplet is pretty much superimposed on another. Those combinations
# are in spots 0, 5, 10 and 15 in our sorted list of keys. This uses a
# boolean and on a byte string representing all possibilities that
# overlap to find that condition, and sets the allowed label positions
# to be diagonally opposite one another for the two droplets. (And it
# colors those numbers blue in the table to make them easy to see.)
bit_status_string = andbytes(
b"1000010000100001",
b"".join([
b"0" if label_overlaps[row_index][x] == 0 else b"1"
for x in range(len(area_list))
]),
)
if bit_status_string == b"1000010000100001":
area_list_string = " ".join([
start_color("bright blue") + area_list[x] +
stop_color()
if b"1000010000100001"[x] == 49 # integer byte value
else area_list[x]
for x in range(len(bit_status_string))
])
area_list_string = (area_list_string +
start_color("bright red") +
" (complete overlap)" + stop_color())
# These two labels have overlaps in all four of their corner areas.
# This gets the corners for the destination label of the vector,
# tuple[1].
# corners = self._pick_corners(
# vector(
# self.labels[id_tuple[0]].center,
# self.labels[id_tuple[1]].center,
# )
# )
# We're setting the corner or corners to avoid to False, so
# we'll get the reversed list.
# TODO Ignoring this for now, until we see if the distance
# TODO approach works to avoid collisions.
# for corner in self._reverse_corners(corners):
# self.labels[id_tuple[0]].corner_status[corner] = False
# # And for the other label, avoid the original list.
# for corner in corners:
# self.labels[id_tuple[1]].corner_status[corner] = False
#
# print("Overlapping labels: {} and {}.".format(*id_tuple)) # Debug
else:
area_list_string = " ".join(area_list)
# print(bit_status_string)
if self._DEBUG:
print(area_list_string)
if self._DEBUG:
print()
#
# Fix label overlaps.
#
# if label_overlaps.sum() != 0: # There are overlaps.
if self.frame == 2:
debug_Catcher = True
# Debug - draw a red line connecting droplets
if self._DEBUG:
self._connect_dots(video_frame)
# Pick a corner for the label, hopelfully one that won't
# interfere with other droplets.
self._choose_label_corners()
if self._DEBUG:
print()
#
# Make changes to and draw labels.
#
for label in self.labels:
# if self.frame == 106:
# debug_catcher = True
# Just to make it easier to type and read...
this_label = self.labels[label]
# If this label doesn't have a corner defined from any previous operations.
if not this_label.corner_used:
# Assign label corner, starting with 0, checking for any
# marked as False by frame edge test or collision code, etc.
for corner in this_label.corner_status:
if this_label.corner_status[corner] is not False:
this_label.corner_used = corner
break
else:
# Uh oh. No available corners. Force a corner
# and complain.
this_label.corner_used = 2
# printc("\n!!!\n!!! Label {} has no available corner!\n!!!"
# .format(this_label.id), 'bright red') # Debug
# For debugging and manual experimentation.
if self._DEBUG:
this_label.draw_all_corner_boxes(video_frame)
# video_frame = this_label.draw_label(video_frame, 0)
# video_frame = this_label.draw_label(video_frame, 1)
# video_frame = this_label.draw_label(video_frame, 2)
# video_frame = this_label.draw_label(video_frame, 3)
video_frame = this_label.draw_label(video_frame,
this_label.corner_used)
return video_frame