def test_i3d(self): i3d = I3DStrategy() video = Video(self.video_path, ExtractorFactory.I3D.value) i3d.extract(video) features = video.features() self.assertIsInstance(features, np.ndarray) assert features.shape == (17, 2048)
def test_slowfast(self): slowfast = SlowFastStrategy() video = Video(self.video_path, ExtractorFactory.SLOWFAST.value) slowfast.extract(video) features = video.features() self.assertIsInstance(features, np.ndarray) assert features.shape == (17, 2304)
class VideoSegmentationCluster(Page): """ Class that represents a the streamlit page of the video segmentation cluster tool Parameters ---------- folder_path: str Path to the folder where the videos a user might want to segment are change_dir_checkbox: bool Boolean that flags if the user ticked a checkbox UI component to change the folder_path selected_filename: str Video's file name selected by the user video_path: str Path to the selected video, it is the folder_path + selected_filename selected_extractor: str Name of the feature extractor selected by the user feature_extractor: ExtractionStrategy ExtractionStrategy object representing the selected feature extractor video: Video Video object representing the video selected by the user selected_cluster: str Name of the cluster strategy selected by the user positional_encoding_checkbox: bool Boolean that flags if the user ticked a checkbox UI component to use positional encoding auto_cluster_checkbox: bool Boolean that flags if the user ticked a checkbox UI component to use the automatic optimal number of clusters algorithm num_cluster_slider: int Number of clusters the user wants to cluster the video cluster: ClusterStrategy ClusterStrategy object representing the selected cluster algorithm features: np.ndarray Numpy array of shape (S, F) with the features of the selected video, where S is the number of segments features this video has been divided into and F is the dimensionality of each feature cluster_labels: np.ndarray Cluster label of each feature segment cluster_df: pd.DataFrame Dataframe with the features after having its dimensionality reduced and its cluster labels """ folder_path = "./data" change_dir_checkbox = None selected_filename = None video_path = None selected_extractor = None feature_extractor = None video = None selected_cluster = None positional_encoding_checkbox = None auto_cluster_checkbox = None num_cluster_slider = None cluster = None features = None cluster_labels = None cluster_df = None def _init_page(self): """ This method calls all component methods in the correct order that they appear in the UI and manages any logical dependency between them. """ self._folder_selection_component() if self.folder_path: self._file_selection_component() if self.video_path: self._feature_extractor_component() self._cluster_selection_component() if self.video_path.split(".")[-1] == 'avi': print(self.video_path) st.video(self.video_path, format='video/avi') else: st.video(self.video_path) self._cluster_plot_component() def _folder_selection_component(self): """ Method that creates the folder selection UI components and their logical dependencies """ # creates checkbox UI component self.change_dir_checkbox = st.sidebar.checkbox("Change directory") if self.change_dir_checkbox: # if checkbox is ticked # creates text_input UI component self.folder_path = st.text_input( "Enter the folder path where your videos are") def _file_selection_component(self): """ Method that creates file selection UI components and their logical dependencies. """ # checking if folder_path is a valid directory if os.path.isdir(self.folder_path) is False: st.error("The path you entered is invalid") else: filenames = os.listdir(self.folder_path) # list all files filenames = [ # Filter only mp4 file names filename for filename in filenames if filename.split(".")[-1] in ["mp4", "avi"] ] # if there are no video in the folder_path user should chose another if len(filenames) == 0: st.error("There are no videos in this folder") else: # Creates a title UI component st.sidebar.title("Pick your video") # creates the selection box component UI and stores the selected # file self.selected_filename = st.sidebar.selectbox( 'Select a file', filenames) self.video_path = os.path.join(self.folder_path, self.selected_filename) def _feature_extractor_component(self): """ Method that creates all feature extraction UI components and their logical dependencies. """ # creates a title UI component st.sidebar.title("Pick a feature extractor") # gets all available extraction strategies available_extraction_strategies = ExtractorFactory.values_list() # creates a selection box UI component with the available_extraction_strategies # and stores the selected in a variable self.selected_extractor = st.sidebar.selectbox( 'Select an extraction strategy', available_extraction_strategies) # gets the feature extractor object from the ExtractorFactory self.feature_extractor = ExtractorFactory.get( self.selected_extractor)() # creates the video object with its path and selected extractor name self.video = Video(self.video_path, self.selected_extractor) # if the video doesn't have features extracted with this extractor # it extracts the features if self.video.features.has_features is False: with st.spinner("Extracting..."): self.feature_extractor.extract(self.video) def _cluster_selection_component(self): """ Method that creates the cluster selection UI components and their logical dependencies. """ # creates a title UI component st.sidebar.title("Cluster information") # gets all available cluster strategies available_cluster_strategies = ClusterFactory.values_list() # creates a selection box UI component with the available_cluster_strategies # and stores the selected cluster strategy in a variable self.selected_cluster = st.sidebar.selectbox( 'Select a cluster strategy', available_cluster_strategies) # creates the positional encoding checkbox self.positional_encoding_checkbox = st.sidebar.checkbox( "Use Positional Encoding") # creates the auto cluster checkbox self.auto_cluster_checkbox = st.sidebar.checkbox( "Automatic find the optimal number of clusters") # if auto cluster is ticked, just initialize it with an integer if self.auto_cluster_checkbox: self.num_cluster_slider = 2 else: # otherwise creates a slider UI component so the user can pick the number of clusters self.num_cluster_slider = st.sidebar.slider( "Select number of clusters", min_value=2, max_value=8) # gets the feature extractor object from the ExtractorFactory self.cluster = ClusterFactory.get( self.selected_cluster)(n=self.num_cluster_slider) # gets the features from the video object self.features = self.video.features() # if the positional encoding checkbox is ticked, preprocess the features # with the positional_encoding function if self.positional_encoding_checkbox: self.features = positional_encoding(self.features) # if auto cluster checkbox is ticked, use the auto method if self.auto_cluster_checkbox: self.cluster_labels = self.cluster.auto(self.features) else: # otherwise the estimate method self.cluster_labels = self.cluster.estimate(self.features) def _cluster_plot_component(self): """ Method that creates the cluster plot UI components and their logical dependencies. """ # creates the features and cluster labels dataframe self.cluster_df = self.cluster.df(self.features, self.cluster_labels) # creates the video timeline chart and scatter chart timeline_chart = plots.video_timeline( cluster_to_segment_bounds(self.cluster_df.labels)) scatter_chart = plots.clusters(self.cluster_df) # creates the altair chart UI component st.altair_chart(alt.vconcat(timeline_chart, scatter_chart))
def _init_videos(self): logger.debug('.') gt_stat = Counter() for root, dirs, files in os.walk(os.path.join(opt.data, opt.subfolder)): if files: for filename in files: # pick only videos with certain complex action # (ex: just concerning coffee) if self._subaction in filename: if opt.test_set: if opt.reduced: opt.reduced = opt.reduced - 1 continue if opt.dataset == 'fs': gt_name = filename[:-(len(opt.ext) + 1)] + '.txt' else: match = re.match(r'(\w*)\.\w*', filename) gt_name = match.group(1) # use extracted features from pretrained on gt embedding if opt.load_embed_feat: path = os.path.join( opt.data, 'embed', opt.subaction, opt.resume_str % opt.subaction) + '_%s' % gt_name else: path = os.path.join(root, filename) start = 0 if self._features is None else self._features.shape[ 0] try: video = Video(path, K=self._K, gt=self.gt_map.gt[gt_name], name=gt_name, start=start, with_bg=self._with_bg) except AssertionError: logger.debug('Assertion Error: %s' % gt_name) continue self._features = join_data(self._features, video.features(), np.vstack) video.reset( ) # to not store second time loaded features self._videos.append(video) # accumulate statistic for inverse counts vector for each video gt_stat.update(self.gt_map.gt[gt_name]) if opt.reduced: if len(self._videos) > opt.reduced: break if opt.feature_dim > 100: if len(self._videos) % 20 == 0: logger.debug('loaded %d videos' % len(self._videos)) # update global range within the current collection for each video for video in self._videos: video.update_indexes(len(self._features)) logger.debug('gt statistic: %d videos ' % len(self._videos) + str(gt_stat)) self._update_fg_mask()