def add_class_results(self, draw, track, frame_offset, rect,
track_predictions, screen_bounds):
prediction = track_predictions[track]
if track not in track_predictions:
return
label = globs._classifier.labels[prediction.label_at_time(
frame_offset)]
score = prediction.score_at_time(frame_offset) * 10
novelty = prediction.novelty_history[frame_offset]
prediction_format = "({:.1f} {})\nnovelty={:.2f}"
current_prediction_string = prediction_format.format(
score * 10, label, novelty)
header_size = self.font_title.getsize(self.track_descs[track])
footer_size = self.font.getsize(current_prediction_string)
# figure out where to draw everything
header_rect = Region(rect.left * self.frame_scale,
rect.top * self.frame_scale - header_size[1],
header_size[0], header_size[1])
footer_center = ((rect.width * self.frame_scale) - footer_size[0]) / 2
footer_rect = Region(rect.left * self.frame_scale + footer_center,
rect.bottom * self.frame_scale, footer_size[0],
footer_size[1])
self.fit_to_image(header_rect, screen_bounds)
self.fit_to_image(footer_rect, screen_bounds)
draw.text((header_rect.x, header_rect.y),
self.track_descs[track],
font=self.font_title)
draw.text((footer_rect.x, footer_rect.y),
current_prediction_string,
font=self.font)
def add_text_to_track(draw,
rect,
header_text,
footer_text,
screen_bounds,
v_offset=0,
scale=1):
font = get_font()
font_title = get_title_font()
header_size = font_title.getsize(header_text)
footer_size = font.getsize(footer_text)
# figure out where to draw everything
header_rect = Region(
rect.left * scale,
(v_offset + rect.top) * scale - header_size[1],
header_size[0],
header_size[1],
)
footer_center = ((rect.width * scale) - footer_size[0]) / 2
footer_rect = Region(
rect.left * scale + footer_center,
(v_offset + rect.bottom) * scale,
footer_size[0],
footer_size[1],
)
fit_to_image(header_rect, screen_bounds)
fit_to_image(footer_rect, screen_bounds)
draw.text((header_rect.x, header_rect.y), header_text, font=font_title)
draw.text((footer_rect.x, footer_rect.y), footer_text, font=font)
def export_clip_preview(self,
filename,
tracker: TrackExtractor,
track_predictions=None):
"""
Exports a clip showing the tracking and predictions for objects within the clip.
"""
# increased resolution of video file.
# videos look much better scaled up
if tracker.stats:
self.auto_max = tracker.stats['max_temp']
self.auto_min = tracker.stats['min_temp']
else:
print("Do not have temperatures to use.")
return
if bool(track_predictions
) and self.preview_type == self.PREVIEW_CLASSIFIED:
self.create_track_descriptions(tracker, track_predictions)
if self.preview_type == self.PREVIEW_TRACKING and not tracker.frame_buffer.flow:
tracker.generate_optical_flow()
mpeg = MPEGCreator(filename)
for frame_number, thermal in enumerate(tracker.frame_buffer.thermal):
if self.preview_type == self.PREVIEW_RAW:
image = self.convert_and_resize(thermal)
if self.preview_type == self.PREVIEW_TRACKING:
image = self.create_four_tracking_image(
tracker.frame_buffer, frame_number)
image = self.convert_and_resize(image, 3.0, mode=Image.NEAREST)
draw = ImageDraw.Draw(image)
regions = tracker.region_history[frame_number]
self.add_regions(draw, regions)
self.add_regions(draw, regions, v_offset=120)
self.add_tracks(draw, tracker.tracks, frame_number)
if self.preview_type == self.PREVIEW_BOXES:
image = self.convert_and_resize(thermal, 4.0)
draw = ImageDraw.Draw(image)
screen_bounds = Region(0, 0, image.width, image.height)
self.add_tracks(draw, tracker.tracks, frame_number)
if self.preview_type == self.PREVIEW_CLASSIFIED:
image = self.convert_and_resize(thermal, 4.0)
draw = ImageDraw.Draw(image)
screen_bounds = Region(0, 0, image.width, image.height)
self.add_tracks(draw, tracker.tracks, frame_number,
track_predictions, screen_bounds)
mpeg.next_frame(np.asarray(image))
# we store the entire video in memory so we need to cap the frame count at some point.
if frame_number > 9 * 60 * 10:
break
mpeg.close()
def add_debug_text(
draw,
track,
region,
screen_bounds,
text=None,
v_offset=0,
frame_offset=0,
scale=1,
):
font = get_font()
if text is None:
text = "id {}".format(track.get_id())
if region.pixel_variance:
text += "mass {} var {} vel ({},{})".format(
region.mass,
round(region.pixel_variance, 2),
track.vel_x[frame_offset],
track.vel_y[frame_offset],
)
footer_size = font.getsize(text)
footer_center = (
(region.width * self.frame_scale) - footer_size[0]) / 2
footer_rect = Region(
region.right * scale - footer_center / 2.0,
(v_offset + region.bottom) * self.frame_scale,
footer_size[0],
footer_size[1],
)
fit_to_image(footer_rect, screen_bounds)
draw.text((footer_rect.x, footer_rect.y), text, font=font)
def smooth(self, frame_bounds: Rectangle):
"""
Smooths out any quick changes in track dimensions
:param frame_bounds The boundaries of the video frame.
"""
if len(self.bounds_history) == 0:
return
new_bounds_history = []
prev_frame = self.bounds_history[0]
current_frame = self.bounds_history[0]
next_frame = self.bounds_history[1]
for i in range(len(self.bounds_history)):
prev_frame = self.bounds_history[max(0, i-1)]
current_frame = self.bounds_history[i]
next_frame = self.bounds_history[min(len(self.bounds_history)-1, i+1)]
frame_x = current_frame.mid_x
frame_y = current_frame.mid_y
frame_width = (prev_frame.width + current_frame.width + next_frame.width) / 3
frame_height = (prev_frame.height + current_frame.height + next_frame.height) / 3
frame = Region(int(frame_x - frame_width / 2), int(frame_y - frame_height / 2), int(frame_width), int(frame_height))
frame.crop(frame_bounds)
new_bounds_history.append(frame)
self.bounds_history = new_bounds_history
def best_trackless_region(clip):
"""Choose a frame for clips without any track"""
best_region = None
THUMBNAIL_SIZE = 64
# if we have regions take best mass of un tracked regions
for regions in clip.region_history:
for region in regions:
if best_region is None or region.mass > best_region.mass:
best_region = region
if best_region is not None:
return best_region
# take region with greatest filtered mean values, and
# if zero take thermal mean values
best_frame_i = np.argmax(clip.stats.frame_stats_mean)
best_frame = clip.frame_buffer.get_frame(best_frame_i).thermal
frame_height, frame_width = best_frame.shape
best_filtered = best_frame - clip.background
best_region = None
for y in range(frame_height - THUMBNAIL_SIZE):
for x in range(frame_width - THUMBNAIL_SIZE):
thermal_sum = np.mean(
best_frame[y : y + THUMBNAIL_SIZE, x : x + THUMBNAIL_SIZE]
)
filtered_sum = np.mean(
best_filtered[y : y + THUMBNAIL_SIZE, x : x + THUMBNAIL_SIZE]
)
if best_region is None:
best_region = ((y, x), filtered_sum, thermal_sum)
elif best_region[1] > 0:
if best_region[1] < filtered_sum:
best_region = ((y, x), thermal_sum, filtered_sum)
elif best_region[2] < thermal_sum:
best_region = ((y, x), thermal_sum, filtered_sum)
return Region(
best_region[0][1],
best_region[0][1],
THUMBNAIL_SIZE,
THUMBNAIL_SIZE,
frame_number=best_frame_i,
)
def add_blank_frame(self, buffer_frame=None):
"""Maintains same bounds as previously, does not reset framce_since_target_seen counter"""
if self.frames > Track.MIN_KALMAN_FRAMES:
region = Region(
int(self.predicted_mid[0] - self.last_bound.width / 2.0),
int(self.predicted_mid[1] - self.last_bound.height / 2.0),
self.last_bound.width,
self.last_bound.height,
)
if self.crop_rectangle:
region.crop(self.crop_rectangle)
else:
region = self.last_bound.copy()
region.blank = True
region.mass = 0
region.pixel_variance = 0
region.frame_number = self.last_bound.frame_number + 1
self.bounds_history.append(region)
self.prev_frame_num = region.frame_number
self.update_velocity()
self.blank_frames += 1
self.frames_since_target_seen += 1
prediction = self.kalman_tracker.predict()
self.predicted_mid = (prediction[0][0], prediction[1][0])
def _get_regions_of_interest(
self, clip, component_details, filtered, prev_filtered
):
"""
Calculates pixels of interest mask from filtered image, and returns both the labeled mask and their bounding
rectangles.
:param filtered: The filtered frame
= :return: regions of interest, mask frame
"""
if prev_filtered is not None:
delta_frame = np.abs(filtered - prev_filtered)
else:
delta_frame = None
# we enlarge the rects a bit, partly because we eroded them previously, and partly because we want some context.
padding = self.frame_padding
# find regions of interest
regions = []
for i, component in enumerate(component_details[1:]):
region = Region(
component[0],
component[1],
component[2],
component[3],
mass=component[4],
id=i,
frame_number=clip.frame_on,
)
old_region = region.copy()
region.crop(clip.crop_rectangle)
region.was_cropped = str(old_region) != str(region)
region.set_is_along_border(clip.crop_rectangle)
if self.config.cropped_regions_strategy == "cautious":
crop_width_fraction = (
old_region.width - region.width
) / old_region.width
crop_height_fraction = (
old_region.height - region.height
) / old_region.height
if crop_width_fraction > 0.25 or crop_height_fraction > 0.25:
continue
elif (
self.config.cropped_regions_strategy == "none"
or self.config.cropped_regions_strategy is None
):
if region.was_cropped:
continue
elif self.config.cropped_regions_strategy != "all":
raise ValueError(
"Invalid mode for CROPPED_REGIONS_STRATEGY, expected ['all','cautious','none'] but found {}".format(
self.config.cropped_regions_strategy
)
)
region.enlarge(padding, max=clip.crop_rectangle)
if delta_frame is not None:
region_difference = region.subimage(delta_frame)
region.pixel_variance = np.var(region_difference)
# filter out regions that are probably just noise
if (
region.pixel_variance < self.config.aoi_pixel_variance
and region.mass < self.config.aoi_min_mass
):
continue
regions.append(region)
return regions
def _get_regions_of_interest(self, clip, labels, stats, thresh, filtered,
prev_filtered, mass):
"""
Calculates pixels of interest mask from filtered image, and returns both the labeled mask and their bounding
rectangles.
:param filtered: The filtered frame
= :return: regions of interest, mask frame
"""
if prev_filtered is not None:
delta_frame = np.abs(filtered - prev_filtered)
else:
delta_frame = None
# we enlarge the rects a bit, partly because we eroded them previously, and partly because we want some context.
padding = self.frame_padding
edge = self.config.edge_pixels
# find regions of interest
regions = []
for i in range(1, labels):
region = Region(
stats[i, 0],
stats[i, 1],
stats[i, 2],
stats[i, 3],
stats[i, 4],
0,
i,
clip.frame_on,
)
# want the real mass calculated from before the dilation
# region.mass = np.sum(region.subimage(thresh))
region.mass = mass
# Add padding to region and change coordinates from edgeless image -> full image
region.x += edge - padding
region.y += edge - padding
region.width += padding * 2
region.height += padding * 2
old_region = region.copy()
region.crop(clip.crop_rectangle)
region.was_cropped = str(old_region) != str(region)
region.set_is_along_border(clip.crop_rectangle)
if self.config.cropped_regions_strategy == "cautious":
crop_width_fraction = (old_region.width -
region.width) / old_region.width
crop_height_fraction = (old_region.height -
region.height) / old_region.height
if crop_width_fraction > 0.25 or crop_height_fraction > 0.25:
continue
elif self.config.cropped_regions_strategy == "none":
if region.was_cropped:
continue
elif self.config.cropped_regions_strategy != "all":
raise ValueError(
"Invalid mode for CROPPED_REGIONS_STRATEGY, expected ['all','cautious','none'] but found {}"
.format(self.config.cropped_regions_strategy))
if delta_frame is not None:
region_difference = region.subimage(delta_frame)
region.pixel_variance = np.var(region_difference)
# filter out regions that are probably just noise
if (region.pixel_variance < self.config.aoi_pixel_variance
and region.mass < self.config.aoi_min_mass):
continue
regions.append(region)
return regions
def export_clip_preview(self, filename, clip: Clip, predictions=None):
"""
Exports a clip showing the tracking and predictions for objects within the clip.
"""
logging.info("creating clip preview %s", filename)
# increased resolution of video file.
# videos look much better scaled up
if not clip.stats:
logging.error("Do not have temperatures to use.")
return
if self.debug:
footer = Previewer.stats_footer(clip.stats)
if predictions and (self.preview_type == self.PREVIEW_CLASSIFIED
or self.preview_type == self.PREVIEW_TRACKING):
self.create_track_descriptions(clip, predictions)
if clip.stats.min_temp is None or clip.stats.max_temp is None:
thermals = [frame.thermal for frame in clip.frame_buffer.frames]
clip.stats.min_temp = np.amin(thermals)
clip.stats.max_temp = np.amax(thermals)
mpeg = MPEGCreator(filename)
frame_scale = 4.0
for frame_number, frame in enumerate(clip.frame_buffer):
if self.preview_type == self.PREVIEW_RAW:
image = self.convert_and_resize(frame.thermal,
clip.stats.min_temp,
clip.stats.max_temp)
draw = ImageDraw.Draw(image)
elif self.preview_type == self.PREVIEW_TRACKING:
image = self.create_four_tracking_image(
frame,
clip.stats.min_temp,
clip.stats.max_temp,
)
draw = ImageDraw.Draw(image)
self.add_tracks(
draw,
clip.tracks,
frame_number,
predictions,
scale=frame_scale,
)
elif self.preview_type == self.PREVIEW_BOXES:
image = self.convert_and_resize(
frame.thermal,
clip.stats.min_temp,
clip.stats.max_temp,
frame_scale=frame_scale,
)
draw = ImageDraw.Draw(image)
screen_bounds = Region(0, 0, image.width, image.height)
self.add_tracks(
draw,
clip.tracks,
frame_number,
colours=[(128, 255, 255)],
scale=frame_scale,
)
elif self.preview_type == self.PREVIEW_CLASSIFIED:
image = self.convert_and_resize(
frame.thermal,
clip.stats.min_temp,
clip.stats.max_temp,
frame_scale=frame_scale,
)
draw = ImageDraw.Draw(image)
screen_bounds = Region(0, 0, image.width, image.height)
self.add_tracks(
draw,
clip.tracks,
frame_number,
predictions,
screen_bounds,
scale=frame_scale,
)
if frame.ffc_affected:
self.add_header(draw, image.width, image.height,
"Calibrating ...")
if self.debug and draw:
self.add_footer(draw, image.width, image.height, footer,
frame.ffc_affected)
mpeg.next_frame(np.asarray(image))
# we store the entire video in memory so we need to cap the frame count at some point.
if frame_number > clip.frames_per_second * 60 * 10:
break
clip.frame_buffer.close_cache()
mpeg.close()
def get_regions_of_interest(self, filtered, prev_filtered=None):
"""
Calculates pixels of interest mask from filtered image, and returns both the labeled mask and their bounding
rectangles.
:param filtered: The filtered frame
:param prev_filtered: The previous filtered frame, required for pixel deltas to be calculated
:return: regions of interest, mask frame
"""
frame_height, frame_width = filtered.shape
# get frames change
if prev_filtered is not None:
# we need a lot of precision because the values are squared. Float32 should work.
delta_frame = np.abs(np.float32(filtered) -
np.float32(prev_filtered))
else:
delta_frame = None
# remove the edges of the frame as we know these pixels can be spurious value
edgeless_filtered = self.crop_rectangle.subimage(filtered)
thresh = np.uint8(blur_and_return_as_mask(edgeless_filtered, threshold=self.threshold))
dilated = thresh
# Dilation groups interested pixels that are near to each other into one component(animal/track)
if self.config.dilation_pixels > 0:
size = self.config.dilation_pixels * 2 + 1
kernel = np.ones((size, size), np.uint8)
dilated = cv2.dilate(dilated, kernel, iterations=1)
labels, small_mask, stats, _ = cv2.connectedComponentsWithStats(dilated)
# make mask go back to full frame size without edges chopped
edge = self.config.edge_pixels
mask = np.zeros(filtered.shape)
mask[edge:frame_height - edge, edge:frame_width - edge] = small_mask
# we enlarge the rects a bit, partly because we eroded them previously, and partly because we want some context.
padding = self.frame_padding
# find regions of interest
regions = []
for i in range(1, labels):
region = Region(
stats[i, 0],
stats[i, 1],
stats[i, 2],
stats[i, 3],
stats[i, 4], 0, i, self.frame_on
)
# want the real mass calculated from before the dilation
region.mass = np.sum(region.subimage(thresh))
# Add padding to region and change coordinates from edgeless image -> full image
region.x += edge - padding
region.y += edge - padding
region.width += padding * 2
region.height += padding * 2
old_region = region.copy()
region.crop(self.crop_rectangle)
region.was_cropped = str(old_region) != str(region)
if self.config.cropped_regions_strategy == "cautious":
crop_width_fraction = (old_region.width - region.width) / old_region.width
crop_height_fraction = (old_region.height - region.height) / old_region.height
if crop_width_fraction > 0.25 or crop_height_fraction > 0.25:
continue
elif self.config.cropped_regions_strategy == "none":
if region.was_cropped:
continue
elif self.config.cropped_regions_strategy != "all":
raise ValueError(
"Invalid mode for CROPPED_REGIONS_STRATEGY, expected ['all','cautious','none'] but found {}".format(
self.config.cropped_regions_strategy))
if delta_frame is not None:
region_difference = np.float32(region.subimage(delta_frame))
region.pixel_variance = np.var(region_difference)
# filter out regions that are probably just noise
if region.pixel_variance < self.config.aoi_pixel_variance and region.mass < self.config.aoi_min_mass:
continue
regions.append(region)
return regions, mask
def remove_background_animals(self, initial_frame, initial_diff):
"""
Try and remove animals that are already in the initial frames, by
checking for connected components in the intital_diff frame
(this is the maximum change between first frame and all other frames in the clip)
"""
# remove some noise
initial_diff[initial_diff < self.background_thresh] = 0
initial_diff[initial_diff > 255] = 255
initial_diff = np.uint8(initial_diff)
initial_diff = cv2.fastNlMeansDenoising(initial_diff, None)
_, lower_mask, lower_objects = detect_objects(initial_diff, otsus=True)
max_region = Region(0, 0, self.res_x, self.res_y)
for component in lower_objects[1:]:
region = Region(component[0], component[1], component[2],
component[3])
region.enlarge(2, max=max_region)
if region.width >= self.res_x or region.height >= self.res_y:
logging.info(
"Background animal bigger than max, probably false positive %s %s",
region,
component[4],
)
continue
background_region = region.subimage(initial_frame)
norm_back = background_region.copy()
norm_back, _ = normalize(norm_back, new_max=255)
sub_components, sub_connected, sub_stats = detect_objects(
norm_back, otsus=True)
if sub_components <= 1:
continue
overlap_image = region.subimage(lower_mask) * 255
overlap_pixels = np.sum(sub_connected[overlap_image > 0])
overlap_pixels = overlap_pixels / float(component[4])
# filter out components which are too big, or dont match original causes
# for filtering
if (overlap_pixels < Clip.MIN_ORIGIN_OVERLAP
or sub_stats[1][4] == 0 or sub_stats[1][4] == region.area):
logging.info(
"Invalid components mass: %s, components: %s region area %s overlap %s",
sub_stats[1][4],
sub_components,
region.area,
overlap_pixels,
)
continue
sub_connected[sub_connected > 0] = 1
# remove this component from the background by painting with
# colours of neighbouring pixels
background_region[:] = cv2.inpaint(
np.float32(background_region),
np.uint8(sub_connected),
3,
cv2.INPAINT_TELEA,
)
return initial_frame