def test_i3d(self):
i3d = I3DStrategy()
video = Video(self.video_path, ExtractorFactory.I3D.value)
i3d.extract(video)
features = video.features()
self.assertIsInstance(features, np.ndarray)
assert features.shape == (17, 2048)
def test_slowfast(self):
slowfast = SlowFastStrategy()
video = Video(self.video_path, ExtractorFactory.SLOWFAST.value)
slowfast.extract(video)
features = video.features()
self.assertIsInstance(features, np.ndarray)
assert features.shape == (17, 2304)
class VideoSegmentationCluster(Page):
"""
Class that represents a the streamlit page of the video segmentation cluster
tool
Parameters
----------
folder_path: str
Path to the folder where the videos a user might want to segment are
change_dir_checkbox: bool
Boolean that flags if the user ticked a checkbox UI component to change
the folder_path
selected_filename: str
Video's file name selected by the user
video_path: str
Path to the selected video, it is the folder_path + selected_filename
selected_extractor: str
Name of the feature extractor selected by the user
feature_extractor: ExtractionStrategy
ExtractionStrategy object representing the selected feature extractor
video: Video
Video object representing the video selected by the user
selected_cluster: str
Name of the cluster strategy selected by the user
positional_encoding_checkbox: bool
Boolean that flags if the user ticked a checkbox UI component to use
positional encoding
auto_cluster_checkbox: bool
Boolean that flags if the user ticked a checkbox UI component to use
the automatic optimal number of clusters algorithm
num_cluster_slider: int
Number of clusters the user wants to cluster the video
cluster: ClusterStrategy
ClusterStrategy object representing the selected cluster algorithm
features: np.ndarray
Numpy array of shape (S, F) with the features of the selected video, where
S is the number of segments features this video has been divided into and
F is the dimensionality of each feature
cluster_labels: np.ndarray
Cluster label of each feature segment
cluster_df: pd.DataFrame
Dataframe with the features after having its dimensionality reduced and
its cluster labels
"""
folder_path = "./data"
change_dir_checkbox = None
selected_filename = None
video_path = None
selected_extractor = None
feature_extractor = None
video = None
selected_cluster = None
positional_encoding_checkbox = None
auto_cluster_checkbox = None
num_cluster_slider = None
cluster = None
features = None
cluster_labels = None
cluster_df = None
def _init_page(self):
""" This method calls all component methods in the correct order that
they appear in the UI and manages any logical dependency between them.
"""
self._folder_selection_component()
if self.folder_path:
self._file_selection_component()
if self.video_path:
self._feature_extractor_component()
self._cluster_selection_component()
if self.video_path.split(".")[-1] == 'avi':
print(self.video_path)
st.video(self.video_path, format='video/avi')
else:
st.video(self.video_path)
self._cluster_plot_component()
def _folder_selection_component(self):
"""
Method that creates the folder selection UI components and their logical
dependencies
"""
# creates checkbox UI component
self.change_dir_checkbox = st.sidebar.checkbox("Change directory")
if self.change_dir_checkbox: # if checkbox is ticked
# creates text_input UI component
self.folder_path = st.text_input(
"Enter the folder path where your videos are")
def _file_selection_component(self):
"""
Method that creates file selection UI components and their logical
dependencies.
"""
# checking if folder_path is a valid directory
if os.path.isdir(self.folder_path) is False:
st.error("The path you entered is invalid")
else:
filenames = os.listdir(self.folder_path) # list all files
filenames = [ # Filter only mp4 file names
filename for filename in filenames
if filename.split(".")[-1] in ["mp4", "avi"]
]
# if there are no video in the folder_path user should chose another
if len(filenames) == 0:
st.error("There are no videos in this folder")
else:
# Creates a title UI component
st.sidebar.title("Pick your video")
# creates the selection box component UI and stores the selected
# file
self.selected_filename = st.sidebar.selectbox(
'Select a file', filenames)
self.video_path = os.path.join(self.folder_path,
self.selected_filename)
def _feature_extractor_component(self):
"""
Method that creates all feature extraction UI components and their logical
dependencies.
"""
# creates a title UI component
st.sidebar.title("Pick a feature extractor")
# gets all available extraction strategies
available_extraction_strategies = ExtractorFactory.values_list()
# creates a selection box UI component with the available_extraction_strategies
# and stores the selected in a variable
self.selected_extractor = st.sidebar.selectbox(
'Select an extraction strategy', available_extraction_strategies)
# gets the feature extractor object from the ExtractorFactory
self.feature_extractor = ExtractorFactory.get(
self.selected_extractor)()
# creates the video object with its path and selected extractor name
self.video = Video(self.video_path, self.selected_extractor)
# if the video doesn't have features extracted with this extractor
# it extracts the features
if self.video.features.has_features is False:
with st.spinner("Extracting..."):
self.feature_extractor.extract(self.video)
def _cluster_selection_component(self):
"""
Method that creates the cluster selection UI components and their logical
dependencies.
"""
# creates a title UI component
st.sidebar.title("Cluster information")
# gets all available cluster strategies
available_cluster_strategies = ClusterFactory.values_list()
# creates a selection box UI component with the available_cluster_strategies
# and stores the selected cluster strategy in a variable
self.selected_cluster = st.sidebar.selectbox(
'Select a cluster strategy', available_cluster_strategies)
# creates the positional encoding checkbox
self.positional_encoding_checkbox = st.sidebar.checkbox(
"Use Positional Encoding")
# creates the auto cluster checkbox
self.auto_cluster_checkbox = st.sidebar.checkbox(
"Automatic find the optimal number of clusters")
# if auto cluster is ticked, just initialize it with an integer
if self.auto_cluster_checkbox:
self.num_cluster_slider = 2
else: # otherwise creates a slider UI component so the user can pick the number of clusters
self.num_cluster_slider = st.sidebar.slider(
"Select number of clusters", min_value=2, max_value=8)
# gets the feature extractor object from the ExtractorFactory
self.cluster = ClusterFactory.get(
self.selected_cluster)(n=self.num_cluster_slider)
# gets the features from the video object
self.features = self.video.features()
# if the positional encoding checkbox is ticked, preprocess the features
# with the positional_encoding function
if self.positional_encoding_checkbox:
self.features = positional_encoding(self.features)
# if auto cluster checkbox is ticked, use the auto method
if self.auto_cluster_checkbox:
self.cluster_labels = self.cluster.auto(self.features)
else: # otherwise the estimate method
self.cluster_labels = self.cluster.estimate(self.features)
def _cluster_plot_component(self):
"""
Method that creates the cluster plot UI components and their logical
dependencies.
"""
# creates the features and cluster labels dataframe
self.cluster_df = self.cluster.df(self.features, self.cluster_labels)
# creates the video timeline chart and scatter chart
timeline_chart = plots.video_timeline(
cluster_to_segment_bounds(self.cluster_df.labels))
scatter_chart = plots.clusters(self.cluster_df)
# creates the altair chart UI component
st.altair_chart(alt.vconcat(timeline_chart, scatter_chart))
def _init_videos(self):
logger.debug('.')
gt_stat = Counter()
for root, dirs, files in os.walk(os.path.join(opt.data,
opt.subfolder)):
if files:
for filename in files:
# pick only videos with certain complex action
# (ex: just concerning coffee)
if self._subaction in filename:
if opt.test_set:
if opt.reduced:
opt.reduced = opt.reduced - 1
continue
if opt.dataset == 'fs':
gt_name = filename[:-(len(opt.ext) + 1)] + '.txt'
else:
match = re.match(r'(\w*)\.\w*', filename)
gt_name = match.group(1)
# use extracted features from pretrained on gt embedding
if opt.load_embed_feat:
path = os.path.join(
opt.data, 'embed', opt.subaction,
opt.resume_str %
opt.subaction) + '_%s' % gt_name
else:
path = os.path.join(root, filename)
start = 0 if self._features is None else self._features.shape[
0]
try:
video = Video(path,
K=self._K,
gt=self.gt_map.gt[gt_name],
name=gt_name,
start=start,
with_bg=self._with_bg)
except AssertionError:
logger.debug('Assertion Error: %s' % gt_name)
continue
self._features = join_data(self._features,
video.features(), np.vstack)
video.reset(
) # to not store second time loaded features
self._videos.append(video)
# accumulate statistic for inverse counts vector for each video
gt_stat.update(self.gt_map.gt[gt_name])
if opt.reduced:
if len(self._videos) > opt.reduced:
break
if opt.feature_dim > 100:
if len(self._videos) % 20 == 0:
logger.debug('loaded %d videos' %
len(self._videos))
# update global range within the current collection for each video
for video in self._videos:
video.update_indexes(len(self._features))
logger.debug('gt statistic: %d videos ' % len(self._videos) +
str(gt_stat))
self._update_fg_mask()
