Example #1
0
    def __init__(self,
                 models=list(),
                 total_norm_weights=None,
                 score_name='fc-action',
                 dev_id=0):
        """
        Contruct an action classifier
        Args:
            models: list of tuples in the form of
                    (model_proto, model_params, model_fusion_weight, input_type, conv_support, input_size).
                    input_type is: 0-RGB, 1-Optical flow.
                    conv_support indicates whether the network supports convolution testing, which is faster. If this is
                    not supported, we will use oversampling instead
            total_norm_weights: sum of all model_fusion_weights when normalization is wanted, otherwise use None
        """

        self.__net_vec = [
            CaffeNet(x[0],
                     x[1],
                     dev_id,
                     input_size=(340, 256) if x[4] else None) for x in models
        ]
        self.__net_weights = [float(x[2]) for x in models]

        if total_norm_weights is not None:
            s = sum(self.__net_weights)
            self.__net_weights = [x / s for x in self.__net_weights]

        self.__input_type = [x[3] for x in models]
        self.__conv_support = [x[4] for x in models]

        self.__num_net = len(models)

        # the input size of the network
        self.__input_size = [x[5] for x in models]

        # whether we should prepare flow stack
        self.__need_flow = max(self.__input_type) > 0

        # the name in the proto for action classes
        self.__score_name = score_name

        # the video downloader
        #self.__video_dl = youtube_dl.YoutubeDL(
        #    {
        #        'outtmpl': '%(id)s.%(ext)s'
        #    }
        #)

        if self.__need_flow:
            self.__flow_extractor = FlowExtractor(dev_id)
    def __init__(self, models=list(), total_norm_weights=None, score_name='fc-action', dev_id=0):
        """
        Contruct an action classifier
        Args:
            models: list of tuples in the form of
                    (model_proto, model_params, model_fusion_weight, input_type, conv_support, input_size).
                    input_type is: 0-RGB, 1-Optical flow.
                    conv_support indicates whether the network supports convolution testing, which is faster. If this is
                    not supported, we will use oversampling instead
            total_norm_weights: sum of all model_fusion_weights when normalization is wanted, otherwise use None
        """

        self.__net_vec = [CaffeNet(x[0], x[1], dev_id,
                                   input_size=(340, 256) if x[4] else None
                                   ) for x in models]
        self.__net_weights = [float(x[2]) for x in models]

        if total_norm_weights is not None:
            s = sum(self.__net_weights)
            self.__net_weights = [x/s for x in self.__net_weights]

        self.__input_type = [x[3] for x in models]
        self.__conv_support = [x[4] for x in models]

        self.__num_net = len(models)

        # the input size of the network
        self.__input_size = [x[5] for x in models]

        # whether we should prepare flow stack
        self.__need_flow = max(self.__input_type) > 0

        # the name in the proto for action classes
        self.__score_name = score_name

        # the video downloader
        self.__video_dl = youtube_dl.YoutubeDL(
            {
                'outtmpl': '%(id)s.%(ext)s'
            }
        )

        if self.__need_flow:
            self.__flow_extractor = FlowExtractor(dev_id)
class ActionClassifier(object):
    """
    This class provides and end-to-end interface to classifying videos into activity classes
    """
    def __init__(self,
                 models=list(),
                 total_norm_weights=None,
                 score_name='fc-action',
                 dev_id=0):
        """
        Contruct an action classifier
        Args:
            models: list of tuples in the form of
                    (model_proto, model_params, model_fusion_weight, input_type, conv_support, input_size).
                    input_type is: 0-RGB, 1-Optical flow.
                    conv_support indicates whether the network supports convolution testing, which is faster. If this is
                    not supported, we will use oversampling instead
            total_norm_weights: sum of all model_fusion_weights when normalization is wanted, otherwise use None
        """

        self.__net_vec = [
            CaffeNet(x[0],
                     x[1],
                     dev_id,
                     input_size=(340, 256) if x[4] else None) for x in models
        ]
        self.__net_weights = [float(x[2]) for x in models]

        if total_norm_weights is not None:
            s = sum(self.__net_weights)
            self.__net_weights = [x / s for x in self.__net_weights]

        self.__input_type = [x[3] for x in models]
        self.__conv_support = [x[4] for x in models]

        self.__num_net = len(models)

        # the input size of the network
        self.__input_size = [x[5] for x in models]

        # whether we should prepare flow stack
        self.__need_flow = max(self.__input_type) > 0

        # the name in the proto for action classes
        self.__score_name = score_name

        # the video downloader
        self.__video_dl = youtube_dl.YoutubeDL({'outtmpl': '%(id)s.%(ext)s'})

        if self.__need_flow:
            self.__flow_extractor = FlowExtractor(dev_id)

    def classify(self, video, model_mask=None, cache_manager=None):
        """

        Args:
            video:

        Returns:
            scores:
            frm_scores:
        """
        import urlparse

        if os.path.isfile(video):
            return self._classify_from_file(video, model_mask, cache_manager)
        elif urlparse.urlparse(video).scheme != "":
            return self._classify_from_url(video, model_mask, cache_manager)

        raise ValueError("Unknown input data type")

    def _classify_from_file(self, filename, model_mask, cache_manager=None):
        """
        Input a file on harddisk
        Args:
            filename:
            cache: cache intermediate results and use previously cached intermediate result is possible

        Returns:
            cls: classification scores
            frm_scores: frame-wise classification scores
        """
        vid_info = _dummy_vid_info()
        vid_info.path = filename
        video_proc = VideoProc(vid_info)
        video_proc.open_video(True)

        # here we use interval of 30, roughly 1FPS
        frm_it = None
        cached_flow = None
        if cache_manager is not None:
            frm_it = cache_manager.load(videoname=filename, type="framestack")
            cached_flow = cache_manager.load(videoname=filename,
                                             type="flowstack")
        if frm_it is None:
            frm_it = video_proc.frame_iter(timely=False,
                                           ignore_err=True,
                                           interval=30,
                                           length=6 if self.__need_flow else 1,
                                           new_size=(340, 256))

        all_scores = []
        all_start = time.clock()

        # process model mask
        mask = [True] * self.__num_net
        n_model = self.__num_net
        if model_mask is not None:
            for i in xrange(len(model_mask)):
                mask[i] = model_mask[i]
                if not mask[i]:
                    n_model -= 1

        frame_cache = []
        flow_cache = []
        cnt = 0
        for frm_stack in frm_it:

            if cache_manager is not None:
                frame_cache.append(frm_stack)

            start = time.clock()
            cnt += 1
            frm_scores = []

            flow_stack = None
            for net, run, in_type, conv_support, net_input_size in \
                    zip(self.__net_vec, mask, self.__input_type, self.__conv_support, self.__input_size):
                if not run:
                    continue

                frame_size = (340 * net_input_size / 224,
                              256 * net_input_size / 224)

                if in_type == 0:
                    # RGB input
                    frm_scores.append(
                        net.predict_single_frame(
                            frm_stack[:1],
                            self.__score_name,
                            over_sample=not conv_support,
                            frame_size=None
                            if net_input_size == 224 else frame_size))
                elif in_type == 1:
                    # Flow input
                    if flow_stack is None:
                        # Extract flow if necessary
                        if cached_flow is not None:
                            flow_stack = cached_flow[cnt - 1]
                        else:
                            flow_stack = self.__flow_extractor.extract_flow(
                                frm_stack, frame_size)
                        if cache_manager is not None:
                            flow_cache.append(flow_stack)

                    frm_scores.append(
                        net.predict_single_flow_stack(
                            flow_stack,
                            self.__score_name,
                            over_sample=not conv_support))
            all_scores.append(frm_scores)
            end = time.clock()
            elapsed = end - start
            # print "frame sample {}: {} second".format(cnt, elapsed)

        if cache_manager is not None:
            if len(frame_cache) != 0:
                cache_manager.dump(frame_cache, filename, "framestack")
            if len(flow_cache) != 0:
                cache_manager.dump(flow_cache, filename, "flowstack")

        # aggregate frame-wise scores
        agg_scores = []
        for i in xrange(n_model):
            model_scores = sliding_window_aggregation_func(np.array(
                [x[i] for x in all_scores]),
                                                           norm=False)
            agg_scores.append(model_scores)

        final_scores = default_fusion_func(
            np.zeros_like(agg_scores[0]), agg_scores,
            [w for w, m in zip(self.__net_weights, mask) if m])

        all_end = time.clock()
        total_time = all_end - all_start
        # print "total time: {} second".format(total_time)
        print('{0} processed.'.format(filename))
        return final_scores, all_scores, total_time

    def _classify_from_url(self, url, model_mask, cache_manager=None):
        """
        This function classify an video based on input video url
        It will first use Youtube-dl to download the video. Then will do classification on the downloaded file
        Returns:
            cls: classification scores
            frm_scores: frame-wise classification scores
        """

        file_info = self.__video_dl.extract_info(
            url)  # it also downloads the video file
        filename = file_info['id'] + '.' + file_info['ext']

        scores, frm_scores, total_time = self._classify_from_file(
            filename, model_mask, cache_manager)
        import os
        os.remove(filename)
        return scores, frm_scores, total_time
Example #4
0
class ActionClassifier(object):
    """
    This class provides and end-to-end interface to classifying videos into activity classes
    """
    def __init__(self,
                 models=list(),
                 total_norm_weights=None,
                 score_name='',
                 dev_id=0):
        """
        Contruct an action classifier
        Args:
            models: list of tuples in the form of
                    (model_proto, model_params, model_fusion_weight, input_type, conv_support, input_size).
                    input_type is: 0-RGB, 1-Optical flow.
                    conv_support indicates whether the network supports convolution testing, which is faster. If this is
                    not supported, we will use oversampling instead
            total_norm_weights: sum of all model_fusion_weights when normalization is wanted, otherwise use None
        """

        self.__net_vec = [
            CaffeNet(x[0],
                     x[1],
                     dev_id,
                     input_size=(340, 256) if x[4] else None) for x in models
        ]
        self.__net_weights = [float(x[2]) for x in models]

        if total_norm_weights is not None:
            s = sum(self.__net_weights)
            self.__net_weights = [x / s for x in self.__net_weights]

        self.__input_type = [x[3] for x in models]
        self.__conv_support = [x[4] for x in models]

        self.__num_net = len(models)

        # the input size of the network
        self.__input_size = [x[5] for x in models]

        # whether we should prepare flow stack
        self.__need_flow = max(self.__input_type) > 0

        # the name in the proto for action classes
        self.__score_name_resnet = 'caffe.Flatten_673'
        self.__score_name_bn = 'global_pool'

        # the video downloader
        self.__video_dl = youtube_dl.YoutubeDL({'outtmpl': '%(id)s.%(ext)s'})

        if self.__need_flow:
            self.__flow_extractor = FlowExtractor(dev_id)

    def classify(self, video, model_mask=None):
        """

        Args:
            video:

        Returns:
            scores:
            all_features:
        """
        import urlparse

        if os.path.isfile(video):
            return self._classify_from_file(video, model_mask)
        elif urlparse.urlparse(video).scheme != "":
            return self._classify_from_url(video, model_mask)

        raise ValueError("Unknown input data type")

    def _classify_from_file(self, filename, model_mask):
        """
        Input a file on harddisk
        Args:
            filename:

        Returns:
            cls: classification scores
            all_features: RGB ResNet feature and Optical flow BN Inception feature in a list
        """

        duration = getLength(filename)
        duration_in_second = float(duration[0][15:17]) * 60 + float(
            duration[0][18:23])
        info_dict = {
            'annotations': list(),
            'url': '',
            'duration': duration_in_second,
            'subset': 'testing'
        }

        vid_info = Video('0', info_dict)
        # update dummy video info...

        vid_info.path = filename
        video_proc = VideoProc(vid_info)
        video_proc.open_video(True)

        # here we use interval of 30, roughly 1FPS
        frm_it = video_proc.frame_iter(timely=True,
                                       ignore_err=True,
                                       interval=0.5,
                                       length=6 if self.__need_flow else 1,
                                       new_size=(340, 256))

        all_features = {
            'resnet': np.empty(shape=(0, 2048)),
            'bn': np.empty(shape=(0, 1024))
        }
        all_start = time.clock()

        cnt = 0

        # process model mask
        mask = [True] * self.__num_net
        n_model = self.__num_net
        if model_mask is not None:
            for i in xrange(len(model_mask)):
                mask[i] = model_mask[i]
                if not mask[i]:
                    n_model -= 1

        for frm_stack in frm_it:

            start = time.clock()
            cnt += 1

            flow_stack = None
            for net, run, in_type, conv_support, net_input_size in \
                    zip(self.__net_vec, mask, self.__input_type, self.__conv_support, self.__input_size):
                if not run:
                    continue

                frame_size = (340 * net_input_size / 224,
                              256 * net_input_size / 224)

                if in_type == 0:
                    # RGB input
                    # TODO for now we only sample one frame w/o applying mean-pooling
                    all_features['resnet'] = np.concatenate(
                        (all_features['resnet'],
                         net.predict_single_frame(
                             frm_stack[:1],
                             self.__score_name_resnet,
                             over_sample=not conv_support,
                             frame_size=None
                             if net_input_size == 224 else frame_size)),
                        axis=0)
                elif in_type == 1:
                    # Flow input
                    if flow_stack is None:
                        # Extract flow if necessary
                        # we disabled spatial data aug
                        # the size for flow frames are 224 x 224, hard coded
                        flow_frame_size = (224, 224)
                        flow_stack = self.__flow_extractor.extract_flow(
                            frm_stack, flow_frame_size)

                    # store all the optical flow features
                    # all_features['bn'] = np.concatenate((all_features['bn'], np.squeeze(net.predict_single_flow_stack(flow_stack, self.__score_name_bn,
                    #                   over_sample=not conv_support))), axis=0)

                    # store only the optical flow feature for the center crop
                    bn_aug = np.squeeze(
                        net.predict_single_flow_stack(flow_stack,
                                                      self.__score_name_bn,
                                                      over_sample=False))
                    # over_sample=not conv_support))
                    # bn_aug = np.squeeze(bn_aug)
                    # bn_center = bn_aug[5]
                    bn_center = bn_aug
                    bn_center = np.reshape(bn_center, (1, bn_center.shape[0]))
                    all_features['bn'] = np.concatenate(
                        (all_features['bn'], bn_center), axis=0)

            end = time.clock()
            elapsed = end - start
            print "frame sample {}: {} second".format(cnt, elapsed)

        print all_features['resnet'].shape, all_features['bn'].shape
        np.save(filename[:-4] + "_resnet", all_features['resnet'])
        np.save(filename[:-4] + "_bn", all_features['bn'])

        return all_features

    def _classify_from_url(self, url, model_mask):
        """
        This function classify an video based on input video url
        It will first use Youtube-dl to download the video. Then will do classification on the downloaded file
        Returns:
            cls: classification scores
            all_features: RGB ResNet feature and Optical flow BN Inception feature in a list
        """

        file_info = self.__video_dl.extract_info(
            url)  # it also downloads the video file
        filename = file_info['id'] + '.' + file_info['ext']

        scores, all_features, total_time = self._classify_from_file(
            filename, model_mask)
        import os
        os.remove(filename)
        return scores, all_features, total_time
class ActionClassifier(object):
    """
    This class provides and end-to-end interface to classifying videos into activity classes
    """

    def __init__(self, models=list(), total_norm_weights=None, score_name='fc-action', dev_id=0):
        """
        Contruct an action classifier
        Args:
            models: list of tuples in the form of
                    (model_proto, model_params, model_fusion_weight, input_type, conv_support, input_size).
                    input_type is: 0-RGB, 1-Optical flow.
                    conv_support indicates whether the network supports convolution testing, which is faster. If this is
                    not supported, we will use oversampling instead
            total_norm_weights: sum of all model_fusion_weights when normalization is wanted, otherwise use None
        """

        self.__net_vec = [CaffeNet(x[0], x[1], dev_id,
                                   input_size=(340, 256) if x[4] else None
                                   ) for x in models]
        self.__net_weights = [float(x[2]) for x in models]

        if total_norm_weights is not None:
            s = sum(self.__net_weights)
            self.__net_weights = [x/s for x in self.__net_weights]

        self.__input_type = [x[3] for x in models]
        self.__conv_support = [x[4] for x in models]

        self.__num_net = len(models)

        # the input size of the network
        self.__input_size = [x[5] for x in models]

        # whether we should prepare flow stack
        self.__need_flow = max(self.__input_type) > 0

        # the name in the proto for action classes
        self.__score_name = score_name

        # the video downloader
        self.__video_dl = youtube_dl.YoutubeDL(
            {
                'outtmpl': '%(id)s.%(ext)s'
            }
        )

        if self.__need_flow:
            self.__flow_extractor = FlowExtractor(dev_id)

    def classify(self, video, model_mask=None):
        """

        Args:
            video:

        Returns:
            scores:
            frm_scores:
        """
        import urlparse

        if os.path.isfile(video):
            return self._classify_from_file(video, model_mask)
        elif urlparse.urlparse(video).scheme != "":
            return self._classify_from_url(video, model_mask)

        raise ValueError("Unknown input data type")

    def _classify_from_file(self, filename, model_mask):
        """
        Input a file on harddisk
        Args:
            filename:

        Returns:
            cls: classification scores
            frm_scores: frame-wise classification scores
        """
        vid_info = _dummy_vid_info()
        vid_info.path = filename
        video_proc = VideoProc(vid_info)
        video_proc.open_video(True)

        # here we use interval of 30, roughly 1FPS
        frm_it = video_proc.frame_iter(timely=False, ignore_err=True, interval=30,
                                       length=6 if self.__need_flow else 1,
                                       new_size=(340, 256))

        all_scores = []
        all_start = time.clock()

        cnt = 0

        # process model mask
        mask = [True] * self.__num_net
        n_model = self.__num_net
        if model_mask is not None:
            for i in xrange(len(model_mask)):
                mask[i] = model_mask[i]
                if not mask[i]:
                    n_model -= 1


        for frm_stack in frm_it:

            start = time.clock()
            cnt += 1
            frm_scores = []

            flow_stack = None
            for net, run, in_type, conv_support, net_input_size in \
                    zip(self.__net_vec, mask, self.__input_type, self.__conv_support, self.__input_size):
                if not run:
                    continue

                frame_size = (340 * net_input_size / 224, 256 * net_input_size / 224)

                if in_type == 0:
                    # RGB input

                    frm_scores.append(net.predict_single_frame(frm_stack[:1], self.__score_name,
                                                               over_sample=not conv_support,
                                                               frame_size=None if net_input_size == 224 else frame_size
                                                               ))
                elif in_type == 1:
                    # Flow input
                    if flow_stack is None:
                        # Extract flow if necessary
                        flow_stack = self.__flow_extractor.extract_flow(frm_stack, frame_size)

                    frm_scores.append(net.predict_single_flow_stack(flow_stack, self.__score_name,
                                                                    over_sample=not conv_support))

            all_scores.append(frm_scores)
            end = time.clock()
            elapsed = end - start
            print "frame sample {}: {} second".format(cnt, elapsed)

        # aggregate frame-wise scores
        agg_scores = []
        for i in xrange(n_model):
            model_scores = sliding_window_aggregation_func(np.array([x[i] for x in all_scores]), norm=False)
            agg_scores.append(model_scores)

        final_scores = default_fusion_func(np.zeros_like(agg_scores[0]), agg_scores, [w for w, m in zip(self.__net_weights, mask) if m])

        all_end = time.clock()
        total_time = all_end - all_start
        print "total time: {} second".format(total_time)

        return final_scores, all_scores, total_time

    def _classify_from_url(self, url, model_mask):
        """
        This function classify an video based on input video url
        It will first use Youtube-dl to download the video. Then will do classification on the downloaded file
        Returns:
            cls: classification scores
            frm_scores: frame-wise classification scores
        """

        file_info = self.__video_dl.extract_info(url) # it also downloads the video file
        filename = file_info['id']+'.'+file_info['ext']

        scores, frm_scores, total_time = self._classify_from_file(filename, model_mask)
        import os
        os.remove(filename)
        return scores, frm_scores, total_time