def fetch_track(self, track: TrackHeader):
"""
Fetches data for an entire track
:param track: the track to fetch
:return: segment data of shape [frames, channels, height, width]
"""
data = self.db.get_track(track.clip_id, track.track_number, 0,
track.frames)
data = preprocess_segment(
data,
reference_level=track.frame_temp_median,
frame_velocity=track.frame_velocity,
encode_frame_offsets_in_flow=self.encode_frame_offsets_in_flow,
default_inset=self.DEFAULT_INSET,
)
return data
def fetch_segment(self, segment: SegmentHeader, augment=False):
"""
Fetches data for segment.
:param segment: The segment header to fetch
:param augment: if true applies data augmentation
:return: segment data of shape [frames, channels, height, width]
"""
segment_width = self.segment_length * segment.track.frames_per_second
# if we are requesting a segment smaller than the default segment size take it from the middle.
unused_frames = segment.frames - segment_width
if unused_frames < 0:
raise Exception(
"Maximum segment size for the dataset is {} frames, but requested {}"
.format(segment.frames, segment_width))
first_frame = segment.start_frame + (unused_frames // 2)
last_frame = segment.start_frame + (unused_frames // 2) + segment_width
if augment and unused_frames > 0:
# jitter first frame
prev_frames = first_frame
post_frames = segment.track.frames - 1 - last_frame
max_jitter = max(5, unused_frames)
jitter = np.clip(np.random.randint(-max_jitter, max_jitter),
-prev_frames, post_frames)
else:
jitter = 0
first_frame += jitter
last_frame += jitter
data = self.db.get_track(segment.clip_id, segment.track_number,
first_frame, last_frame)
if len(data) != segment_width:
logging.error("invalid segment length %d, expected %d", len(data),
len(segment_width))
data = preprocess_segment(
data,
segment.track.frame_temp_median[first_frame:last_frame],
segment.track.frame_velocity[first_frame:last_frame],
augment=augment,
default_inset=self.DEFAULT_INSET,
)
return data
def identify_last_frame(self):
"""
Runs through track identifying segments, and then returns it's prediction of what kind of animal this is.
One prediction will be made for every active_track of the last frame.
:return: TrackPrediction object
"""
prediction_smooth = 0.1
smooth_prediction = None
smooth_novelty = None
prediction = 0.0
novelty = 0.0
active_tracks = self.get_active_tracks()
frame = self.clip.frame_buffer.get_last_frame()
if frame is None:
return
thermal_reference = np.median(frame.thermal)
for i, track in enumerate(active_tracks):
track_prediction = self.predictions.get_or_create_prediction(
track, keep_all=False)
region = track.bounds_history[-1]
if region.frame_number != frame.frame_number:
logging.warning(
"frame doesn't match last frame {} and {}".format(
region.frame_number, frame.frame_number))
else:
track_data = track.crop_by_region(frame, region)
# we use a tighter cropping here so we disable the default 2 pixel inset
frames = preprocess_segment([track_data], [thermal_reference],
default_inset=0)
if frames is None:
logging.warning(
"Frame {} of track could not be classified.".format(
region.frame_number))
continue
p_frame = frames[0]
(
prediction,
novelty,
state,
) = self.classifier.classify_frame_with_novelty(
p_frame, track_prediction.state)
track_prediction.state = state
if self.fp_index is not None:
prediction[self.fp_index] *= 0.8
state *= 0.98
mass = region.mass
mass_weight = np.clip(mass / 20, 0.02, 1.0)**0.5
cropped_weight = 0.7 if region.was_cropped else 1.0
prediction *= mass_weight * cropped_weight
if len(track_prediction.predictions) == 0:
if track_prediction.uniform_prior:
smooth_prediction = np.ones([self.num_labels
]) * (1 / self.num_labels)
else:
smooth_prediction = prediction
smooth_novelty = 0.5
else:
smooth_prediction = track_prediction.predictions[-1]
smooth_novelty = track_prediction.novelties[-1]
smooth_prediction = (
1 - prediction_smooth
) * smooth_prediction + prediction_smooth * prediction
smooth_novelty = (
1 - prediction_smooth
) * smooth_novelty + prediction_smooth * novelty
track_prediction.classified_frame(self.clip.frame_on,
smooth_prediction,
smooth_novelty)
def identify_track(self, clip: Clip, track: Track):
"""
Runs through track identifying segments, and then returns it's prediction of what kind of animal this is.
One prediction will be made for every frame.
:param track: the track to identify.
:return: TrackPrediction object
"""
# go through making classifications at each frame
# note: we should probably be doing this every 9 frames or so.
state = None
if self.kerasmodel:
track_prediction = self.classifier.classify_track(clip, track)
self.predictions.prediction_per_track[
track.get_id()] = track_prediction
else:
track_prediction = self.predictions.get_or_create_prediction(track)
for i, region in enumerate(track.bounds_history):
frame = clip.frame_buffer.get_frame(region.frame_number)
track_data = track.crop_by_region(frame, region)
# note: would be much better for the tracker to store the thermal references as it goes.
# frame = clip.frame_buffer.get_frame(frame_number)
thermal_reference = np.median(frame.thermal)
# track_data = track.crop_by_region_at_trackframe(frame, i)
if i % self.FRAME_SKIP == 0:
# we use a tighter cropping here so we disable the default 2 pixel inset
frames = preprocess_segment([track_data],
[thermal_reference],
default_inset=0)
if frames is None:
logging.info(
"Frame {} of track could not be classified.".
format(region.frame_number))
continue
frame = frames[0]
(
prediction,
novelty,
state,
) = self.classifier.classify_frame_with_novelty(
frame, state)
# make false-positive prediction less strong so if track has dead footage it won't dominate a strong
# score
# a little weight decay helps the model not lock into an initial impression.
# 0.98 represents a half life of around 3 seconds.
state *= 0.98
# precondition on weight, segments with small mass are weighted less as we can assume the error is
# higher here.
mass = region.mass
# we use the square-root here as the mass is in units squared.
# this effectively means we are giving weight based on the diameter
# of the object rather than the mass.
mass_weight = np.clip(mass / 20, 0.02, 1.0)**0.5
# cropped frames don't do so well so restrict their score
cropped_weight = 0.7 if region.was_cropped else 1.0
track_prediction.classified_frame(
region.frame_number,
prediction,
mass_scale=mass_weight * cropped_weight,
novelty=novelty,
)
return track_prediction