def handle_connection(connection, config, thermal_config_file): headers, extra_b = handle_headers(connection) thermal_config = ThermalConfig.load_from_file(thermal_config_file, headers.model) logging.info("parsed camera headers %s running with config %s", headers, thermal_config) process_queue = multiprocessing.Queue() processor = get_processor(process_queue, config, thermal_config, headers) processor.start() edge = config.tracking.edge_pixels crop_rectangle = tools.Rectangle(edge, edge, headers.res_x - 2 * edge, headers.res_y - 2 * edge) raw_frame = lepton3.Lepton3(headers) read = 0 try: while True: if extra_b is not None: data = extra_b + connection.recv( headers.frame_size - len(extra_b), socket.MSG_WAITALL) extra_b = None else: data = connection.recv(headers.frame_size, socket.MSG_WAITALL) if not data: logging.info("disconnected from camera") process_queue.put(STOP_SIGNAL) break try: message = data[:5].decode("utf-8") if message == "clear": logging.info("processing error from camera") process_queue.put(STOP_SIGNAL) break except: pass read += 1 frame = raw_frame.parse(data) frame.received_at = time.time() cropped_frame = crop_rectangle.subimage(frame.pix) t_max = np.amax(cropped_frame) t_min = np.amin(cropped_frame) # seems to happen if pi is working hard if t_min == 0: logging.warning( "received frame has odd values skipping thermal frame max {} thermal frame min {} cpu % {} memory % {}" .format(t_max, t_min, psutil.cpu_percent(), psutil.virtual_memory()[2])) process_queue.put(SKIP_SIGNAL) elif read < 100: process_queue.put(SKIP_SIGNAL) else: process_queue.put(frame) finally: time.sleep(5) # give it a moment to close down properly processor.terminate()
def calculate_frame_crop(self): # frames are always square, but bounding rect may not be, so to see how much we clipped I need to create a square # bounded rect and check it against frame size. self.frame_crop = [] for rect in self.regions: rx, ry = rect.mid_x, rect.mid_y size = max(rect.width, rect.height) adjusted_rect = tools.Rectangle(rx - size / 2, ry - size / 2, size, size) self.frame_crop.append( get_cropped_fraction(adjusted_rect, self.res_x, self.res_y))
def __init__(self, config, thermal_config, classifier): self.frame_num = 0 self.clip = None self.tracking = False self.enable_per_track_information = False self.rolling_track_classify = {} self.skip_classifying = 0 self.classified_consec = 0 self.config = config self.classifier = classifier self.num_labels = len(classifier.labels) self._res_x = self.config.res_x self._res_y = self.config.res_y self.predictions = Predictions(classifier.labels) self.preview_frames = (thermal_config.recorder.preview_secs * thermal_config.recorder.frame_rate) edge = self.config.tracking.edge_pixels self.crop_rectangle = tools.Rectangle(edge, edge, self.res_x - 2 * edge, self.res_y - 2 * edge) try: self.fp_index = self.classifier.labels.index("false-positive") except ValueError: self.fp_index = None self.track_extractor = ClipTrackExtractor( self.config.tracking, self.config.use_opt_flow, self.config.classify.cache_to_disk, keep_frames=False, calc_stats=False, ) self.motion_config = thermal_config.motion self.min_frames = (thermal_config.recorder.min_secs * thermal_config.recorder.frame_rate) self.max_frames = (thermal_config.recorder.max_secs * thermal_config.recorder.frame_rate) self.motion_detector = MotionDetector( self.res_x, self.res_y, thermal_config, self.config.tracking.dynamic_thresh, CPTVRecorder(thermal_config), ) self.startup_classifier() self._output_dir = thermal_config.recorder.output_dir self.meta_dir = os.path.join(thermal_config.recorder.output_dir, "metadata") if not os.path.exists(self.meta_dir): os.makedirs(self.meta_dir)
def calculate_frame_crop(self): # frames are always square, but bounding rect may not be, so to see how much we clipped I need to create a square # bounded rect and check it against frame size. self.frame_crop = [] for rect in self.track_bounds: rect = tools.Rectangle.from_ltrb(*rect) rx, ry = rect.mid_x, rect.mid_y size = max(rect.width, rect.height) adjusted_rect = tools.Rectangle(rx - size / 2, ry - size / 2, size, size) self.frame_crop.append( get_cropped_fraction(adjusted_rect, CPTV_FILE_WIDTH, CPTV_FILE_HEIGHT))
def from_meta(clip_id, clip_meta, track_meta): """ Creates a track header from given metadata. """ # kind of checky way to get camera name from clip_id, in the future camera will be included in the metadata. camera = os.path.splitext(os.path.basename(clip_id))[0].split('-')[-1] # get the reference levels from clip_meta and load them into the track. track_start_frame = track_meta['start_frame'] track_end_frame = track_meta['start_frame'] + int(round(track_meta['duration']*9)) thermal_reference_level = np.float32(clip_meta['frame_temp_median'][track_start_frame:track_end_frame]) # calculate the frame velocities bounds_history = track_meta['bounds_history'] frame_center = [((left + right)/2, (top + bottom)/2) for left, top, right, bottom in bounds_history] frame_velocity = [] prev = None for x, y in frame_center: if prev is None: frame_velocity.append((0.0,0.0)) else: frame_velocity.append((x-prev[0], y-prev[1])) prev = (x, y) result = TrackHeader( clip_id=clip_id, track_number=track_meta['id'], label=track_meta['tag'], start_time=parser.parse(track_meta['start_time']), duration=float(track_meta['duration']), camera=camera, score=float(track_meta['score']) ) result.thermal_reference_level = thermal_reference_level result.frame_velocity = frame_velocity result.track_bounds = np.asarray(bounds_history) # frames are always square, but bounding rect may not be, so to see how much we clipped I need to create a square # bounded rect and check it against frame size. result.frame_crop = [] for rect in result.track_bounds: rect = tools.Rectangle.from_ltrb(*rect) rx, ry = rect.mid_x, rect.mid_y size = max(rect.width, rect.height) adjusted_rect = tools.Rectangle(rx-size/2, ry-size/2, size, size) result.frame_crop.append(get_cropped_fraction(adjusted_rect, CPTV_FILE_WIDTH, CPTV_FILE_HEIGHT)) return result
def preprocess_segment( frames, reference_level=None, frame_velocity=None, augment=False, encode_frame_offsets_in_flow=False, default_inset=2, filter_to_delta=True, keep_aspect=False, frame_size=48, ): """ Preprocesses the raw track data, scaling it to correct size, and adjusting to standard levels :param frames: a list of np array of shape [C, H, W] :param reference_level: thermal reference level for each frame in data :param frame_velocity: velocity (x,y) for each frame. :param augment: if true applies a slightly random crop / scale :param default_inset: the default number of pixels to inset when no augmentation is applied. :param filter_to_delta: If true change filterted channel to be the delta of thermal frames. """ if reference_level: # ------------------------------------------- # next adjust temperature and flow levels # get reference level for thermal channel assert len(frames) == len( reference_level ), "Reference level shape and data shape not match." # ------------------------------------------- # first we scale to the standard size # adjusting the corners makes the algorithm robust to tracking differences. top_offset = random.randint(0, 5) if augment else default_inset bottom_offset = random.randint(0, 5) if augment else default_inset left_offset = random.randint(0, 5) if augment else default_inset right_offset = random.randint(0, 5) if augment else default_inset scaled_frames = [] for i, frame in enumerate(frames): channels = frame.shape[0] frame_height, frame_width = frame[0].shape if frame_height < MIN_SIZE or frame_width < MIN_SIZE: continue frame_bounds = tools.Rectangle(0, 0, frame_width, frame_height) # set up a cropping frame crop_region = tools.Rectangle.from_ltrb( left_offset, top_offset, frame_width - right_offset, frame_height - bottom_offset, ) # if the frame is too small we make it a little larger while crop_region.width < MIN_SIZE: crop_region.left -= 1 crop_region.right += 1 crop_region.crop(frame_bounds) while crop_region.height < MIN_SIZE: crop_region.top -= 1 crop_region.bottom += 1 crop_region.crop(frame_bounds) cropped_frame = frame[ :, crop_region.top : crop_region.bottom, crop_region.left : crop_region.right, ] scaled_frame = [ resize_frame(cropped_frame[channel], channel, frame_size, keep_aspect) for channel in range(channels) ] if reference_level: scaled_frame[0] -= np.float32(reference_level[i]) scaled_frames.append(scaled_frame) # convert back into [F,C,H,W] array. data = np.float32(scaled_frames) if len(data) == 0: return None # map optical flow down to right level, # we pre-multiplied by 256 to fit into a 16bit int data[:, 2 : 3 + 1, :, :] *= 1.0 / 256.0 # write frame motion into center of frame if encode_frame_offsets_in_flow: F, C, H, W = data.shape for x in range(-2, 2 + 1): for y in range(-2, 2 + 1): data[:, 2 : 3 + 1, H // 2 + y, W // 2 + x] = frame_velocity[:, :] # set filtered track to delta frames if filter_to_delta: reference = np.clip(data[:, 0], 20, 999) data[0, 1] = 0 data[1:, 1] = reference[1:] - reference[:-1] # ------------------------------------------- # finally apply and additional augmentation if augment: if random.random() <= 0.75: # we will adjust contrast and levels, but only within these bounds. # that is a bright input may have brightness reduced, but not increased. LEVEL_OFFSET = 4 # apply level and contrast shift level_adjust = random.normalvariate(0, LEVEL_OFFSET) contrast_adjust = tools.random_log(0.9, (1 / 0.9)) data[:, 0] *= contrast_adjust data[:, 0] += level_adjust data[:, 1] *= contrast_adjust if random.random() <= 0.50: # when we flip the frame remember to flip the horizontal velocity as well data = np.flip(data, axis=3) data[:, 2] = -data[:, 2] np.clip(data[:, 0, :, :], a_min=0, a_max=None, out=data[:, 0, :, :]) return data
def get_cropped_fraction(region: tools.Rectangle, width, height): """ Returns the fraction regions mass outside the rect ((0,0), (width, height)""" bounds = tools.Rectangle(0, 0, width - 1, height - 1) return 1 - (bounds.overlap_area(region) / region.area)
def preprocess_segment( frames, frame_size, reference_level=None, frame_velocity=None, augment=False, default_inset=0, keep_edge=False, ): """ Preprocesses the raw track data, scaling it to correct size, and adjusting to standard levels :param frames: a list of Frames :param reference_level: thermal reference level for each frame in data :param frame_velocity: velocity (x,y) for each frame. :param augment: if true applies a slightly random crop / scale :param default_inset: the default number of pixels to inset when no augmentation is applied. """ if reference_level is not None: # ------------------------------------------- # next adjust temperature and flow levels # get reference level for thermal channel assert len(frames) == len( reference_level), "Reference level shape and data shape not match." crop_rectangle = tools.Rectangle(EDGE, EDGE, res_x - 2 * EDGE, res_y - 2 * EDGE) # ------------------------------------------- # first we scale to the standard size data = [] flip = False chance = random.random() if augment: contrast_adjust = None level_adjust = None if chance <= 0.75: # we will adjust contrast and levels, but only within these bounds. # that is a bright input may have brightness reduced, but not increased. LEVEL_OFFSET = 4 # apply level and contrast shift level_adjust = float(random.normalvariate(0, LEVEL_OFFSET)) contrast_adjust = float(tools.random_log(0.9, (1 / 0.9))) if chance <= 0.50: flip = True for i, frame in enumerate(frames): frame.float_arrays() frame_height, frame_width = frame.thermal.shape # adjusting the corners makes the algorithm robust to tracking differences. # gp changed to 0,1 maybe should be a percent of the frame size max_height_offset = int(np.clip(frame_height * 0.1, 1, 2)) max_width_offset = int(np.clip(frame_width * 0.1, 1, 2)) top_offset = (int(random.random() * max_height_offset) if augment else default_inset) bottom_offset = (int(random.random() * max_height_offset) if augment else default_inset) left_offset = (int(random.random() * max_width_offset) if augment else default_inset) right_offset = (int(random.random() * max_width_offset) if augment else default_inset) if frame_height < MIN_SIZE or frame_width < MIN_SIZE: continue frame_bounds = tools.Rectangle(0, 0, frame_width, frame_height) # rotate then crop if augment and chance <= 0.75: # degress = 0 degrees = int(chance * 40) - 20 frame.rotate(degrees) # set up a cropping frame crop_region = tools.Rectangle.from_ltrb( left_offset, top_offset, frame_width - right_offset, frame_height - bottom_offset, ) # if the frame is too small we make it a little larger while crop_region.width < MIN_SIZE: crop_region.left -= 1 crop_region.right += 1 crop_region.crop(frame_bounds) while crop_region.height < MIN_SIZE: crop_region.top -= 1 crop_region.bottom += 1 crop_region.crop(frame_bounds) frame.crop_by_region(crop_region, out=frame) # if frame.mask is not None: # assert np.all(np.mod(frame.mask, 1) == 0), "Mask isn't integer" try: frame.resize_with_aspect((frame_size, frame_size), crop_rectangle, keep_edge=keep_edge) except Exception as e: logging.error("Error resizing frame %s exception %s", frame, e) continue if reference_level is not None: frame.thermal -= reference_level[i] np.clip(frame.thermal, a_min=0, a_max=None, out=frame.thermal) frame.normalize() if augment: if level_adjust is not None: frame.brightness_adjust(level_adjust) if contrast_adjust is not None: frame.contrast_adjust(contrast_adjust) if flip: frame.flip() data.append(frame) return data, flip
def apply(frames, reference_level, frame_velocity=None, augment=False, encode_frame_offsets_in_flow=False, default_inset=2): """ Preprocesses the raw track data, scaling it to correct size, and adjusting to standard levels :param frames: a list of np array of shape [C, H, W] :param reference_level: thermal reference level for each frame in data :param frame_velocity: velocity (x,y) for each frame. :param augment: if true applies a slightly random crop / scale :param default_inset: the default number of pixels to inset when no augmentation is applied. """ # ------------------------------------------- # first we scale to the standard size # adjusting the corners makes the algorithm robust to tracking differences. top_offset = random.randint(0, 5) if augment else default_inset bottom_offset = random.randint(0, 5) if augment else default_inset left_offset = random.randint(0, 5) if augment else default_inset right_offset = random.randint(0, 5) if augment else default_inset scaled_frames = [] for frame in frames: channels, frame_height, frame_width = frame.shape frame_bounds = tools.Rectangle(0, 0, frame_width, frame_height) # set up a cropping frame crop_region = tools.Rectangle.from_ltrb(left_offset, top_offset, frame_width - right_offset, frame_height - bottom_offset) # if the frame is too small we make it a little larger while crop_region.width < 4: crop_region.left -=1 crop_region.right += 1 crop_region.crop(frame_bounds) while crop_region.height < 4: crop_region.top -=1 crop_region.bottom += 1 crop_region.crop(frame_bounds) cropped_frame = frame[:, crop_region.top: crop_region.bottom, crop_region.left: crop_region.right] scaled_frame = [cv2.resize(np.float32(cropped_frame[channel]), dsize=(Preprocessor.FRAME_SIZE, Preprocessor.FRAME_SIZE), interpolation=cv2.INTER_LINEAR if channel != TrackChannels.mask else cv2.INTER_NEAREST) for channel in range(channels)] scaled_frame = np.float32(scaled_frame) scaled_frames.append(scaled_frame) # convert back into [F,C,H,W] array. data = np.float32(scaled_frames) # the segment will be processed in float32 so we may as well convert it here. # also optical flow is stored as a scaled integer, but we want it in float32 format. data = np.asarray(data, dtype=np.float32) # ------------------------------------------- # next adjust temperature and flow levels # get reference level for thermal channel assert len(data) == len(reference_level), "Reference level shape and data shape not match." # reference thermal levels to the reference level data[:, 0, :, :] -= np.float32(reference_level)[:, np.newaxis, np.newaxis] # map optical flow down to right level, # we pre-multiplied by 256 to fit into a 16bit int data[:, 2:3 + 1, :, :] *= (1.0/256.0) # write frame motion into center of frame if encode_frame_offsets_in_flow: F, C, H, W = data.shape for x in range(-2,2+1): for y in range(-2,2+1): data[:, 2:3 + 1, H//2+y, W//2+x] = frame_velocity[:, :] # set filtered track to delta frames reference = np.clip(data[:, 0], 20, 999) data[0, 1] = 0 data[1:, 1] = reference[1:] - reference[:-1] # ------------------------------------------- # finally apply and additional augmentation if augment: if (random.random() <= 0.75): # we will adjust contrast and levels, but only within these bounds. # that is a bright input may have brightness reduced, but not increased. LEVEL_OFFSET = 4 # apply level and contrast shift level_adjust = random.normalvariate(0, LEVEL_OFFSET) contrast_adjust = tools.random_log(0.9, (1/0.9)) data[:, 0] *= contrast_adjust data[:, 0] += level_adjust if random.random() <= 0.50: # when we flip the frame remember to flip the horizontal velocity as well data = np.flip(data, axis=3) data[:, 2] = -data[:, 2] return data