Beispiel #1
0
class PiClassifier(Processor):
    """ Classifies frames from leptond """

    PROCESS_FRAME = 3
    NUM_CONCURRENT_TRACKS = 1
    DEBUG_EVERY = 100
    MAX_CONSEC = 3
    # after every MAX_CONSEC frames skip this many frames
    # this gives the cpu a break
    SKIP_FRAMES = 7

    def __init__(self, config, thermal_config, classifier):
        self.frame_num = 0
        self.clip = None
        self.tracking = False
        self.enable_per_track_information = False
        self.rolling_track_classify = {}
        self.skip_classifying = 0
        self.classified_consec = 0
        self.config = config
        self.classifier = classifier
        self.num_labels = len(classifier.labels)
        self._res_x = self.config.res_x
        self._res_y = self.config.res_y
        self.predictions = Predictions(classifier.labels)
        self.preview_frames = (thermal_config.recorder.preview_secs *
                               thermal_config.recorder.frame_rate)
        edge = self.config.tracking.edge_pixels
        self.crop_rectangle = tools.Rectangle(edge, edge,
                                              self.res_x - 2 * edge,
                                              self.res_y - 2 * edge)

        try:
            self.fp_index = self.classifier.labels.index("false-positive")
        except ValueError:
            self.fp_index = None

        self.track_extractor = ClipTrackExtractor(
            self.config.tracking,
            self.config.use_opt_flow,
            self.config.classify.cache_to_disk,
            keep_frames=False,
            calc_stats=False,
        )
        self.motion_config = thermal_config.motion
        self.min_frames = (thermal_config.recorder.min_secs *
                           thermal_config.recorder.frame_rate)
        self.max_frames = (thermal_config.recorder.max_secs *
                           thermal_config.recorder.frame_rate)
        self.motion_detector = MotionDetector(
            self.res_x,
            self.res_y,
            thermal_config,
            self.config.tracking.dynamic_thresh,
            CPTVRecorder(thermal_config),
        )
        self.startup_classifier()

        self._output_dir = thermal_config.recorder.output_dir
        self.meta_dir = os.path.join(thermal_config.recorder.output_dir,
                                     "metadata")
        if not os.path.exists(self.meta_dir):
            os.makedirs(self.meta_dir)

    def new_clip(self):
        self.clip = Clip(self.config.tracking, "stream")
        self.clip.video_start_time = datetime.now()
        self.clip.num_preview_frames = self.preview_frames

        self.clip.set_res(self.res_x, self.res_y)
        self.clip.set_frame_buffer(
            self.config.classify_tracking.high_quality_optical_flow,
            self.config.classify.cache_to_disk,
            self.config.use_opt_flow,
            True,
        )

        # process preview_frames
        frames = self.motion_detector.thermal_window.get_frames()
        for frame in frames:
            self.track_extractor.process_frame(self.clip, frame.copy())

    def startup_classifier(self):
        # classifies an empty frame to force loading of the model into memory

        p_frame = np.zeros((5, 48, 48), np.float32)
        self.classifier.classify_frame_with_novelty(p_frame, None)

    def get_active_tracks(self):
        """
        Gets current clips active_tracks and returns the top NUM_CONCURRENT_TRACKS order by priority
        """
        active_tracks = self.clip.active_tracks
        if len(active_tracks) <= PiClassifier.NUM_CONCURRENT_TRACKS:
            return active_tracks
        active_predictions = []
        for track in active_tracks:
            prediction = self.predictions.get_or_create_prediction(
                track, keep_all=False)
            active_predictions.append(prediction)

        top_priority = sorted(
            active_predictions,
            key=lambda i: i.get_priority(self.clip.frame_on),
            reverse=True,
        )

        top_priority = [
            track.track_id
            for track in top_priority[:PiClassifier.NUM_CONCURRENT_TRACKS]
        ]
        classify_tracks = [
            track for track in active_tracks if track.get_id() in top_priority
        ]
        return classify_tracks

    def identify_last_frame(self):
        """
        Runs through track identifying segments, and then returns it's prediction of what kind of animal this is.
        One prediction will be made for every active_track of the last frame.
        :return: TrackPrediction object
        """

        prediction_smooth = 0.1

        smooth_prediction = None
        smooth_novelty = None

        prediction = 0.0
        novelty = 0.0
        active_tracks = self.get_active_tracks()
        frame = self.clip.frame_buffer.get_last_frame()
        if frame is None:
            return
        thermal_reference = np.median(frame.thermal)

        for i, track in enumerate(active_tracks):
            track_prediction = self.predictions.get_or_create_prediction(
                track, keep_all=False)
            region = track.bounds_history[-1]
            if region.frame_number != frame.frame_number:
                logging.warning(
                    "frame doesn't match last frame {} and {}".format(
                        region.frame_number, frame.frame_number))
            else:
                track_data = track.crop_by_region(frame, region)
                # we use a tighter cropping here so we disable the default 2 pixel inset
                frames = Preprocessor.apply([track_data], [thermal_reference],
                                            default_inset=0)
                if frames is None:
                    logging.warning(
                        "Frame {} of track could not be classified.".format(
                            region.frame_number))
                    continue
                p_frame = frames[0]
                (
                    prediction,
                    novelty,
                    state,
                ) = self.classifier.classify_frame_with_novelty(
                    p_frame, track_prediction.state)
                track_prediction.state = state

                if self.fp_index is not None:
                    prediction[self.fp_index] *= 0.8
                state *= 0.98
                mass = region.mass
                mass_weight = np.clip(mass / 20, 0.02, 1.0)**0.5
                cropped_weight = 0.7 if region.was_cropped else 1.0

                prediction *= mass_weight * cropped_weight

                if len(track_prediction.predictions) == 0:
                    if track_prediction.uniform_prior:
                        smooth_prediction = np.ones([self.num_labels
                                                     ]) * (1 / self.num_labels)
                    else:
                        smooth_prediction = prediction
                    smooth_novelty = 0.5
                else:
                    smooth_prediction = track_prediction.predictions[-1]
                    smooth_novelty = track_prediction.novelties[-1]
                    smooth_prediction = (
                        1 - prediction_smooth
                    ) * smooth_prediction + prediction_smooth * prediction
                    smooth_novelty = (
                        1 - prediction_smooth
                    ) * smooth_novelty + prediction_smooth * novelty
                track_prediction.classified_frame(self.clip.frame_on,
                                                  smooth_prediction,
                                                  smooth_novelty)

    def get_recent_frame(self):
        return self.motion_detector.get_recent_frame()

    def disconnected(self):
        self.end_clip()
        self.motion_detector.disconnected()

    def skip_frame(self):
        self.skip_classifying -= 1

        if self.clip:
            self.clip.frame_on += 1

    def process_frame(self, lepton_frame):
        start = time.time()
        self.motion_detector.process_frame(lepton_frame)
        if self.motion_detector.recorder.recording:
            if self.clip is None:
                self.new_clip()
            self.track_extractor.process_frame(
                self.clip, lepton_frame.pix, self.motion_detector.ffc_affected)
            if self.motion_detector.ffc_affected or self.clip.on_preview():
                self.skip_classifying = PiClassifier.SKIP_FRAMES
                self.classified_consec = 0
            elif (self.motion_detector.ffc_affected is False
                  and self.clip.active_tracks and self.skip_classifying <= 0
                  and not self.clip.on_preview()):
                self.identify_last_frame()
                self.classified_consec += 1
                if self.classified_consec == PiClassifier.MAX_CONSEC:
                    self.skip_classifying = PiClassifier.SKIP_FRAMES
                    self.classified_consec = 0

        elif self.clip is not None:
            self.end_clip()

        self.skip_classifying -= 1
        self.frame_num += 1
        end = time.time()
        timetaken = end - start
        if (self.motion_detector.can_record()
                and self.frame_num % PiClassifier.DEBUG_EVERY == 0):
            logging.info(
                "fps {}/sec time to process {}ms cpu % {} memory % {}".format(
                    round(1 / timetaken, 2),
                    round(timetaken * 1000, 2),
                    psutil.cpu_percent(),
                    psutil.virtual_memory()[2],
                ))

    def create_mp4(self):
        previewer = Previewer(self.config, "classified")
        previewer.export_clip_preview(self.clip.get_id() + ".mp4", self.clip,
                                      self.predictions)

    def end_clip(self):
        if self.clip:
            for _, prediction in self.predictions.prediction_per_track.items():
                if prediction.max_score:
                    logging.info("Clip {} {}".format(
                        self.clip.get_id(),
                        prediction.description(self.predictions.labels),
                    ))
            self.save_metadata()
            self.predictions.clear_predictions()
            self.clip = None
            self.tracking = False

    def save_metadata(self):
        filename = datetime.now().strftime("%Y%m%d.%H%M%S.%f.meta")

        # record results in text file.
        save_file = {}
        start, end = self.clip.start_and_end_time_absolute()
        save_file["start_time"] = start.isoformat()
        save_file["end_time"] = end.isoformat()
        save_file["temp_thresh"] = self.clip.temp_thresh
        save_file["algorithm"] = {}
        save_file["algorithm"]["model"] = self.config.classify.model
        save_file["algorithm"]["tracker_version"] = self.clip.VERSION
        save_file["tracks"] = []
        for track in self.clip.tracks:
            track_info = {}
            prediction = self.predictions.prediction_for(track.get_id())
            start_s, end_s = self.clip.start_and_end_in_secs(track)
            save_file["tracks"].append(track_info)
            track_info["start_s"] = round(start_s, 2)
            track_info["end_s"] = round(end_s, 2)
            track_info["num_frames"] = track.frames
            track_info["frame_start"] = track.start_frame
            track_info["frame_end"] = track.end_frame
            if prediction and prediction.best_label_index is not None:
                track_info["label"] = self.classifier.labels[
                    prediction.best_label_index]
                track_info["confidence"] = round(prediction.score(), 2)
                track_info["clarity"] = round(prediction.clarity, 3)
                track_info["average_novelty"] = round(
                    prediction.average_novelty, 2)
                track_info["max_novelty"] = round(prediction.max_novelty, 2)
                track_info["all_class_confidences"] = {}
                for i, value in enumerate(prediction.class_best_score):
                    label = self.classifier.labels[i]
                    track_info["all_class_confidences"][label] = round(
                        float(value), 3)

            positions = []
            for region in track.bounds_history:
                track_time = round(
                    region.frame_number / self.clip.frames_per_second, 2)
                positions.append([track_time, region])
            track_info["positions"] = positions

        with open(os.path.join(self.meta_dir, filename), "w") as f:
            json.dump(save_file, f, indent=4, cls=tools.CustomJSONEncoder)

    @property
    def res_x(self):
        return self._res_x

    @property
    def res_y(self):
        return self._res_y

    @property
    def output_dir(self):
        return self._output_dir
Beispiel #2
0
class ClipClassifier(CPTVFileProcessor):
    """ Classifies tracks within CPTV files. """

    # skips every nth frame.  Speeds things up a little, but reduces prediction quality.
    FRAME_SKIP = 1

    def __init__(self, config, tracking_config, model_file):
        """ Create an instance of a clip classifier"""

        super(ClipClassifier, self).__init__(config, tracking_config)
        self.model_file = model_file

        # prediction record for each track
        self.predictions = Predictions(self.classifier.labels)

        self.previewer = Previewer.create_if_required(config,
                                                      config.classify.preview)

        self.start_date = None
        self.end_date = None
        self.cache_to_disk = self.config.classify.cache_to_disk
        # enables exports detailed information for each track.  If preview mode is enabled also enables track previews.
        self.enable_per_track_information = False
        self.track_extractor = ClipTrackExtractor(
            self.config.tracking,
            self.config.use_opt_flow
            or config.classify.preview == Previewer.PREVIEW_TRACKING,
            self.config.classify.cache_to_disk,
        )

    def preprocess(self, frame, thermal_reference):
        """
        Applies preprocessing to frame required by the model.
        :param frame: numpy array of shape [C, H, W]
        :return: preprocessed numpy array
        """

        # note, would be much better if the model did this, as only the model knows how preprocessing occurred during
        # training
        frame = np.float32(frame)
        frame[2:3 + 1] *= 1 / 256
        frame[0] -= thermal_reference

        return frame

    def identify_track(self, clip: Clip, track: Track):
        """
        Runs through track identifying segments, and then returns it's prediction of what kind of animal this is.
        One prediction will be made for every frame.
        :param track: the track to identify.
        :return: TrackPrediction object
        """

        # uniform prior stats start with uniform distribution.  This is the safest bet, but means that
        # it takes a while to make predictions.  When off the first prediction is used instead causing
        # faster, but potentially more unstable predictions.
        UNIFORM_PRIOR = False

        num_labels = len(self.classifier.labels)
        prediction_smooth = 0.1

        smooth_prediction = None
        smooth_novelty = None

        prediction = 0.0
        novelty = 0.0
        try:
            fp_index = self.classifier.labels.index("false-positive")
        except ValueError:
            fp_index = None

        # go through making classifications at each frame
        # note: we should probably be doing this every 9 frames or so.
        state = None
        track_prediction = self.predictions.get_or_create_prediction(track)
        for i, region in enumerate(track.bounds_history):
            frame = clip.frame_buffer.get_frame(region.frame_number)
            track_data = track.crop_by_region(frame, region)

            # note: would be much better for the tracker to store the thermal references as it goes.
            # frame = clip.frame_buffer.get_frame(frame_number)
            thermal_reference = np.median(frame.thermal)
            # track_data = track.crop_by_region_at_trackframe(frame, i)
            if i % self.FRAME_SKIP == 0:
                # we use a tighter cropping here so we disable the default 2 pixel inset
                frames = Preprocessor.apply([track_data], [thermal_reference],
                                            default_inset=0)

                if frames is None:
                    logging.info(
                        "Frame {} of track could not be classified.".format(
                            region.frame_number))
                    return
                frame = frames[0]
                (
                    prediction,
                    novelty,
                    state,
                ) = self.classifier.classify_frame_with_novelty(frame, state)
                # make false-positive prediction less strong so if track has dead footage it won't dominate a strong
                # score
                if fp_index is not None:
                    prediction[fp_index] *= 0.8

                # a little weight decay helps the model not lock into an initial impression.
                # 0.98 represents a half life of around 3 seconds.
                state *= 0.98

                # precondition on weight,  segments with small mass are weighted less as we can assume the error is
                # higher here.
                mass = region.mass

                # we use the square-root here as the mass is in units squared.
                # this effectively means we are giving weight based on the diameter
                # of the object rather than the mass.
                mass_weight = np.clip(mass / 20, 0.02, 1.0)**0.5

                # cropped frames don't do so well so restrict their score
                cropped_weight = 0.7 if region.was_cropped else 1.0

                prediction *= mass_weight * cropped_weight

            if smooth_prediction is None:
                if UNIFORM_PRIOR:
                    smooth_prediction = np.ones([num_labels
                                                 ]) * (1 / num_labels)
                else:
                    smooth_prediction = prediction
                smooth_novelty = 0.5
            else:
                smooth_prediction = (
                    1 - prediction_smooth
                ) * smooth_prediction + prediction_smooth * prediction
                smooth_novelty = (
                    1 - prediction_smooth
                ) * smooth_novelty + prediction_smooth * novelty
            track_prediction.classified_frame(region.frame_number,
                                              smooth_prediction,
                                              smooth_novelty)
        return track_prediction

    @property
    def classifier(self):
        """
        Returns a classifier object, which is created on demand.
        This means if the ClipClassifier is copied to a new process a new Classifier instance will be created.
        """
        if globs._classifier is None:
            t0 = datetime.now()
            logging.info("classifier loading")
            globs._classifier = Model(
                train_config=self.config.train,
                session=tools.get_session(disable_gpu=not self.config.use_gpu),
            )
            globs._classifier.load(self.model_file)
            logging.info("classifier loaded ({})".format(datetime.now() - t0))
        return globs._classifier

    def get_meta_data(self, filename):
        """ Reads meta-data for a given cptv file. """
        source_meta_filename = os.path.splitext(filename)[0] + ".txt"
        if os.path.exists(source_meta_filename):

            meta_data = tools.load_clip_metadata(source_meta_filename)

            tags = set()
            for record in meta_data["Tags"]:
                # skip automatic tags
                if record.get("automatic", False):
                    continue
                else:
                    tags.add(record["animal"])

            tags = list(tags)

            if len(tags) == 0:
                tag = "no tag"
            elif len(tags) == 1:
                tag = tags[0] if tags[0] else "none"
            else:
                tag = "multi"
            meta_data["primary_tag"] = tag
            return meta_data
        else:
            return None

    def get_classify_filename(self, input_filename):
        return os.path.splitext(
            os.path.join(self.config.classify.classify_folder,
                         os.path.basename(input_filename)))[0]

    def process_file(self, filename, **kwargs):
        """
        Process a file extracting tracks and identifying them.
        :param filename: filename to process
        :param enable_preview: if true an MPEG preview file is created.
        """

        if not os.path.exists(filename):
            raise Exception("File {} not found.".format(filename))

        logging.info("Processing file '{}'".format(filename))

        start = time.time()
        clip = Clip(self.tracker_config, filename)
        self.track_extractor.parse_clip(clip)

        classify_name = self.get_classify_filename(filename)
        destination_folder = os.path.dirname(classify_name)

        if not os.path.exists(destination_folder):
            logging.info("Creating folder {}".format(destination_folder))
            os.makedirs(destination_folder)

        mpeg_filename = classify_name + ".mp4"

        meta_filename = classify_name + ".txt"

        logging.info(os.path.basename(filename) + ":")

        for i, track in enumerate(clip.tracks):
            prediction = self.identify_track(clip, track)
            description = prediction.description(self.classifier.labels)
            logging.info(" - [{}/{}] prediction: {}".format(
                i + 1, len(clip.tracks), description))

        if self.previewer:
            logging.info("Exporting preview to '{}'".format(mpeg_filename))
            self.previewer.export_clip_preview(mpeg_filename, clip,
                                               self.predictions)
        logging.info("saving meta data")
        self.save_metadata(filename, meta_filename, clip)
        self.predictions.clear_predictions()

        if self.tracker_config.verbose:
            ms_per_frame = ((time.time() - start) * 1000 /
                            max(1, len(clip.frame_buffer.frames)))
            logging.info("Took {:.1f}ms per frame".format(ms_per_frame))

    def save_metadata(self, filename, meta_filename, clip):
        if self.cache_to_disk:
            clip.frame_buffer.remove_cache()

        # read in original metadata
        meta_data = self.get_meta_data(filename)

        # record results in text file.
        save_file = {}
        save_file["source"] = filename
        start, end = clip.start_and_end_time_absolute()
        save_file["start_time"] = start.isoformat()
        save_file["end_time"] = end.isoformat()
        save_file["algorithm"] = {}
        save_file["algorithm"]["model"] = self.model_file
        save_file["algorithm"]["tracker_version"] = clip.VERSION
        save_file["algorithm"]["tracker_config"] = self.tracker_config.as_dict(
        )
        if meta_data:
            save_file["camera"] = meta_data["Device"]["devicename"]
            save_file["cptv_meta"] = meta_data
            save_file["original_tag"] = meta_data["primary_tag"]
        save_file["tracks"] = []
        for track in clip.tracks:
            track_info = {}
            prediction = self.predictions.prediction_for(track.get_id())
            start_s, end_s = clip.start_and_end_in_secs(track)
            save_file["tracks"].append(track_info)
            track_info["start_s"] = round(start_s, 2)
            track_info["end_s"] = round(end_s, 2)
            track_info["num_frames"] = prediction.num_frames
            track_info["frame_start"] = track.start_frame
            track_info["frame_end"] = track.end_frame
            track_info["label"] = self.classifier.labels[
                prediction.best_label_index]
            track_info["confidence"] = round(prediction.score(), 2)
            track_info["clarity"] = round(prediction.clarity, 3)
            track_info["average_novelty"] = round(prediction.average_novelty,
                                                  2)
            track_info["max_novelty"] = round(prediction.max_novelty, 2)
            track_info["all_class_confidences"] = {}
            for i, value in enumerate(prediction.class_best_score):
                label = self.classifier.labels[i]
                track_info["all_class_confidences"][label] = round(
                    float(value), 3)

            positions = []
            for region in track.bounds_history:
                track_time = round(
                    region.frame_number / clip.frames_per_second, 2)
                positions.append([track_time, region])
            track_info["positions"] = positions

        if self.config.classify.meta_to_stdout:
            print(json.dumps(save_file, cls=tools.CustomJSONEncoder))
        else:
            with open(meta_filename, "w") as f:
                json.dump(save_file, f, indent=4, cls=tools.CustomJSONEncoder)