def convert_and_resize(self, image, size = None, mode = Image.BILINEAR):
""" Converts the image to colour using colour map and resize """
image = tools.convert_heat_to_img(image, self.colormap, self.auto_min, self.auto_max)
if size:
self.frame_scale = size
image = image.resize((int(image.width * self.frame_scale), int(image.height * self.frame_scale)), mode)
return image
def export_track_mpeg_previews(self, filename_base, tracker: TrackExtractor):
"""
Exports preview MPEG for a specific track
:param filename_base:
:param tracker:
:param track:
:return:
"""
# resolution of video file.
# videos look much better scaled up
FRAME_SIZE = 4*48
frame_width, frame_height = FRAME_SIZE, FRAME_SIZE
frame_width = frame_width // 4 * 4
frame_height = frame_height // 4 * 4
for id, track in enumerate(tracker.tracks):
video_frames = []
for frame_number in range(len(track.bounds_history)):
channels = tracker.get_track_channels(track, frame_number)
img = tools.convert_heat_to_img(channels[1], self.colormap, 0, 350)
img = img.resize((frame_width, frame_height), Image.NEAREST)
video_frames.append(np.asarray(img))
tools.write_mpeg(filename_base+"-"+str(id+1)+".mp4", video_frames)
def create_individual_track_previews(self, filename, clip: Clip):
# resolution of video file.
# videos look much better scaled up
filename_format = path.splitext(filename)[0] + "-{}.mp4"
FRAME_SIZE = 4 * 48
frame_width, frame_height = FRAME_SIZE, FRAME_SIZE
for id, track in enumerate(clip.tracks):
video_frames = []
for region in track.bounds_history:
frame = clip.frame_buffer.get_frame(region.frame_number)
cropped = frame.crop_by_region(region)
if cropped.thermal.size == 0:
continue
img = tools.convert_heat_to_img(
cropped.thermal,
self.colourmap,
np.amin(cropped.thermal),
np.amax(cropped.thermal),
)
img = img.resize((frame_width, frame_height), Image.NEAREST)
video_frames.append(np.asarray(img))
logging.info("creating preview %s", filename_format.format(id + 1))
tools.write_mpeg(filename_format.format(id + 1), video_frames)
def export_clip_preview(self, filename, tracker: TrackExtractor):
"""
Exports a clip showing the tracking and predictions for objects within the clip.
"""
# increased resolution of video file.
# videos look much better scaled up
FRAME_SCALE = 4.0
NORMALISATION_SMOOTH = 0.95
auto_min = np.min(tracker.frames[0])
auto_max = np.max(tracker.frames[0])
# setting quality to 30 gives files approximately the same size as the original CPTV MPEG previews
# (but they look quite compressed)
mpeg = MPEGCreator(filename)
for frame_number, thermal in enumerate(tracker.frames):
auto_min = NORMALISATION_SMOOTH * auto_min + (
1 - NORMALISATION_SMOOTH) * np.min(thermal)
auto_max = NORMALISATION_SMOOTH * auto_max + (
1 - NORMALISATION_SMOOTH) * np.max(thermal)
thermal_image = convert_heat_to_img(thermal, self.colormap,
auto_min, auto_max)
thermal_image = thermal_image.resize(
(int(thermal_image.width * FRAME_SCALE),
int(thermal_image.height * FRAME_SCALE)), Image.BILINEAR)
if tracker.filtered_frames:
if self.enable_side_by_side:
# put thermal & tracking images side by side
tracking_image = self.export_tracking_frame(
tracker, frame_number, FRAME_SCALE)
side_by_side_image = Image.new(
'RGB',
(tracking_image.width * 2, tracking_image.height))
side_by_side_image.paste(thermal_image, (0, 0))
side_by_side_image.paste(tracking_image,
(tracking_image.width, 0))
mpeg.next_frame(np.asarray(side_by_side_image))
else:
# overlay track rectanges on original thermal image
thermal_image = self.draw_track_rectangles(
tracker, frame_number, FRAME_SCALE, thermal_image)
mpeg.next_frame(np.asarray(thermal_image))
else:
# no filtered frames available (clip too hot or
# background moving?) so just output the original
# frame without the tracking frame.
mpeg.next_frame(np.asarray(thermal_image))
# we store the entire video in memory so we need to cap the frame count at some point.
if frame_number > 9 * 60 * 10:
break
mpeg.close()
def create_four_tracking_image(self, frame, min_temp, max_temp):
thermal = frame.thermal
filtered = frame.filtered + min_temp
filtered = tools.convert_heat_to_img(filtered, self.colourmap,
min_temp, max_temp)
thermal = tools.convert_heat_to_img(thermal, self.colourmap, min_temp,
max_temp)
if self.debug:
tools.add_heat_number(thermal, frame.thermal, 1)
if frame.mask is None:
mask = np.zeros((np.array(thermal).shape), dtype=np.uint8)
else:
mask, _ = normalize(frame.mask, new_max=255)
mask = np.uint8(mask)
mask = mask[..., np.newaxis]
mask = np.repeat(mask, 3, axis=2)
mask = Image.fromarray(mask)
flow_h, flow_v = frame.get_flow_split(clip_flow=True)
if flow_h is None and flow_v is None:
flow_magnitude = Image.fromarray(
np.zeros((np.array(thermal).shape), dtype=np.uint8))
else:
flow_magnitude = (
np.linalg.norm(np.float32([flow_h, flow_v]), ord=2, axis=0) /
4.0 + min_temp)
flow_magnitude = tools.convert_heat_to_img(flow_magnitude,
self.colourmap,
min_temp, max_temp)
image = np.hstack((np.vstack(
(thermal, mask)), np.vstack((filtered, flow_magnitude))))
image = Image.fromarray(image)
image = image.resize(
(
int(image.width * 4),
int(image.height * 4),
),
Image.BILINEAR,
)
return image
def export_tracking_frame(self, tracker, frame_number, frame_scale):
filtered = tracker.filtered_frames[frame_number]
filtered = 3 * filtered + TrackExtractor.TEMPERATURE_MIN
tracking_image = convert_heat_to_img(filtered, self.colormap,
tracker.TEMPERATURE_MIN,
tracker.TEMPERATURE_MAX)
tracking_image = tracking_image.resize(
(int(tracking_image.width * frame_scale),
int(tracking_image.height * frame_scale)), Image.NEAREST)
return self.draw_track_rectangles(tracker, frame_number, frame_scale,
tracking_image)
def add_last_frame_tracking(
frame,
tracks,
track_predictions=None,
screen_bounds=None,
colours=Previewer.TRACK_COLOURS,
tracks_text=None,
v_offset=0,
scale=1,
debug=False,
):
image = tools.convert_heat_to_img(frame.thermal)
draw = ImageDraw.Draw(image)
# look for any tracks that occur on this frame
for index, track in enumerate(tracks):
region = track.bounds_history[-1]
if region.frame_number != frame.frame_number:
continue
draw.rectangle(
rect_points(region, v_offset, scale=scale),
outline=colours[index % len(colours)],
)
if track_predictions is not None:
track_prediction = track_predictions.prediction_for(track)
if track_prediction:
footer_text = track_prediction.get_classified_footer(labels)
add_text_to_track(
draw,
region,
str(track.get_id()),
footer_text,
screen_bounds,
v_offset,
)
if debug:
text = None
if tracks_text and len(tracks_text) > index:
text = tracks_text[index]
add_debug_text(draw,
track,
region,
screen_bounds,
text=text,
v_offset=v_offset)
return image
def __next__(self):
if self.current_frame >= len(self.tracker.frame_buffer):
raise StopIteration
thermal = self.tracker.frame_buffer.thermal[self.current_frame]
filtered = self.tracker.frame_buffer.filtered[self.current_frame]
mask = self.tracker.frame_buffer.mask[self.current_frame]
flow = self.tracker.frame_buffer.flow[self.current_frame]
regions = self.tracker.region_history[self.current_frame]
# marked is an image with each pixel's value being the label, 0...n for n objects
# I multiply it here, but really I should use a seperate color map for this.
# maybe I could multiply it modulo, and offset by some amount?
# This really should be using a pallete here, I multiply by 10000 to make sure the binary mask '1' values get set to the brightest color (which is about 4000)
# here I map the flow magnitude [ranges in the single didgits) to a temperature in the display range.
flow_magnitude = np.linalg.norm(np.float32(flow), ord=2, axis=2) / 4.0
stacked = np.hstack((np.vstack((thermal, mask * 10000)),
np.vstack((3 * filtered + tools.TEMPERATURE_MIN, flow_magnitude + tools.TEMPERATURE_MIN))))
img = tools.convert_heat_to_img(stacked, self.colormap, tools.TEMPERATURE_MIN, tools.TEMPERATURE_MAX)
img = img.resize((int(img.width * self.FRAME_SCALE), int(img.height * self.FRAME_SCALE)), Image.NEAREST)
draw = ImageDraw.Draw(img)
# look for any regions of interest that occur on this frame
for rect in regions:
rect_points = [int(p * self.FRAME_SCALE) for p in
[rect.left, rect.top, rect.right, rect.top, rect.right, rect.bottom, rect.left, rect.bottom,
rect.left, rect.top]]
draw.line(rect_points, (128, 128, 128))
# look for any tracks that occur on this frame
for id, track in enumerate(self.tracker.tracks):
frame_offset = self.current_frame - track.start_frame
if frame_offset >= 0 and frame_offset < len(track.bounds_history) - 1:
# display the track
rect = track.bounds_history[frame_offset]
rect_points = [int(p * self.FRAME_SCALE) for p in
[rect.left, rect.top, rect.right, rect.top, rect.right, rect.bottom, rect.left,
rect.bottom, rect.left, rect.top]]
draw.line(rect_points, self.track_colors[id % len(self.track_colors)])
self.current_frame += 1
return np.asarray(img)
def export_track_preview(self, filename, track):
"""
Exports a clip showing tracking of one specific track with point in time predictions.
"""
preview_scale = 4.0
predictions = self.track_prediction[track].prediction_history
mpeg = MPEGCreator(filename)
for i in range(track.frames):
# export a MPEG preview of the track
frame = track.get_frame(i)
draw_frame = np.float16(frame[:, :, 1])
img = convert_heat_to_img(draw_frame, self.colormap, 0, 300)
img = img.resize((int(
img.width * preview_scale), int(img.height * preview_scale)),
Image.NEAREST)
# just in case we don't have as many predictions as frames.
if i >= len(predictions):
continue
# draw predictions
prediction = predictions[i]
best_labels = np.argsort(-prediction)[:3]
width, height = img.width, img.height
for i, label in enumerate(best_labels):
draw = ImageDraw.Draw(img)
score = prediction[label]
x = 10
y = height - 100 + 10 + i * 30
draw.rectangle([x, y + 16, width - 10, y + 26],
outline=(0, 0, 0),
fill=(0, 64, 0, 64))
draw.rectangle([x, y + 16, 10 + score * (width - 20), y + 26],
outline=(0, 0, 0),
fill=(64, 255, 0, 250))
draw.text([x, y],
self.classifier.classes[label],
font=self.font)
mpeg.next_frame(np.asarray(img))
mpeg.close()
def create_individual_track_previews(self, filename, tracker:TrackExtractor):
# resolution of video file.
# videos look much better scaled up
filename_format = os.path.splitext(filename)[0] + "-{}.mp4"
FRAME_SIZE = 4*48
frame_width, frame_height = FRAME_SIZE, FRAME_SIZE
for id, track in enumerate(tracker.tracks):
video_frames = []
for frame_number in range(len(track.bounds_history)):
channels = tracker.get_track_channels(track, frame_number)
img = tools.convert_heat_to_img(channels[1], self.colormap, 0, 350)
img = img.resize((frame_width, frame_height), Image.NEAREST)
video_frames.append(np.asarray(img))
print("creating preview {}".format(filename_format.format(id + 1)))
tools.write_mpeg(filename_format.format(id + 1), video_frames)
def export_tracking_frame(self, tracker: TrackExtractor, frame_number: int,
frame_scale: float):
mask = tracker.frame_buffer.mask[frame_number]
filtered = tracker.frame_buffer.filtered[frame_number]
tracking_image = tools.convert_heat_to_img(filtered / 200,
self.colormap,
temp_min=0,
temp_max=1)
tracking_image = tracking_image.resize(
(int(tracking_image.width * frame_scale),
int(tracking_image.height * frame_scale)), Image.NEAREST)
return self.draw_track_rectangles(tracker, frame_number, frame_scale,
tracking_image)
def convert_and_resize(self,
frame,
h_min,
h_max,
size=None,
mode=Image.BILINEAR,
frame_scale=1):
"""Converts the image to colour using colour map and resize"""
thermal = frame[:120, :160].copy()
image = tools.convert_heat_to_img(frame, self.colourmap, h_min, h_max)
image = image.resize(
(
int(image.width * frame_scale),
int(image.height * frame_scale),
),
mode,
)
if self.debug:
tools.add_heat_number(image, thermal, frame_scale)
return image
