def scenes(self, threads):
g = nx.Graph()
# connect adjacent shots
for shot1, shot2 in pairwise(threads.itertracks()):
g.add_edge(shot1, shot2)
# connect threaded shots
for label in threads.labels():
for shot1, shot2 in pairwise(threads.subset([label]).itertracks()):
g.add_edge(shot1, shot2)
scenes = threads.copy()
# group all shots of intertwined threads
for shots in sorted(sorted(bc) for bc in nx.biconnected_components(g)):
if len(shots) < 3:
continue
common_label = scenes[shots[0]]
for shot in shots:
scenes[shot] = common_label
return scenes
def scenes(self, threads):
g = nx.Graph()
# connect adjacent shots
for shot1, shot2 in pairwise(threads.itertracks()):
g.add_edge(shot1, shot2)
# connect threaded shots
for label in threads.labels():
for shot1, shot2 in pairwise(threads.subset([label]).itertracks()):
g.add_edge(shot1, shot2)
scenes = threads.copy()
# group all shots of intertwined threads
for shots in sorted(sorted(bc) for bc in nx.biconnected_components(g)):
if len(shots) < 3:
continue
common_label = scenes[shots[0]]
for shot in shots:
scenes[shot] = common_label
return scenes
def predict(self, features, min_duration=None, constraint=None):
"""
Parameters
----------
min_duration : float or dict, optional
Minimum duration for each label, in seconds.
"""
constraint_ = self._constraint(constraint, features)
consecutive = self._consecutive(min_duration, features)
X = self.X(features, unknown="keep")
sliding_window = features.sliding_window
converted_y = self.classifier_.predict(X, consecutive=consecutive, constraint=constraint_)
annotation = Annotation()
diff = list(np.where(np.diff(converted_y))[0])
diff = [-1] + diff + [len(converted_y)]
for t, T in pairwise(diff):
segment = sliding_window.rangeToSegment(t, T - t)
annotation[segment] = converted_y[t + 1]
translation = self.label_converter_.inverse_mapping()
return annotation.translate(translation)
def apply(self, features, segmentation=None):
"""
Parameters
----------
features : Features
segmentation : Timeline, optional
"""
if segmentation is None:
segmentation = Timeline(segments=[features.getExtent()])
sliding_window = features.sliding_window
min_samples = sliding_window.durationToSamples(self.min_duration)
precision = sliding_window.durationToSamples(self.precision)
segmenter = SKLearnBICSegmentation(
penalty_coef=self.penalty_coef,
covariance_type=self.covariance_type,
min_samples=min_samples,
precision=precision)
result = Timeline()
for long_segment in segmentation:
X = features.crop(long_segment)
boundaries = segmenter.apply(X)
for t, T in pairwise(boundaries):
segment = sliding_window.rangeToSegment(t, T - t)
shifted_segment = Segment(long_segment.start + segment.start,
long_segment.start + segment.end)
result.add(shifted_segment)
return result
def apply(self, features, segmentation=None):
"""
Parameters
----------
features : Features
segmentation : Timeline, optional
"""
if segmentation is None:
segmentation = Timeline(segments=[features.getExtent()])
sliding_window = features.sliding_window
min_samples = sliding_window.durationToSamples(self.min_duration)
precision = sliding_window.durationToSamples(self.precision)
segmenter = SKLearnBICSegmentation(
penalty_coef=self.penalty_coef,
covariance_type=self.covariance_type,
min_samples=min_samples,
precision=precision)
result = Timeline()
for long_segment in segmentation:
X = features.crop(long_segment)
boundaries = segmenter.apply(X)
for t, T in pairwise(boundaries):
segment = sliding_window.rangeToSegment(t, T - t)
shifted_segment = Segment(long_segment.start + segment.start,
long_segment.start + segment.end)
result.add(shifted_segment)
return result
def predict(self, features, min_duration=None, constraint=None):
"""
Parameters
----------
min_duration : float or dict, optional
Minimum duration for each label, in seconds.
"""
constraint_ = self._constraint(constraint, features)
consecutive = self._consecutive(min_duration, features)
X = self.X(features, unknown='keep')
sliding_window = features.sliding_window
converted_y = self.classifier_.predict(X,
consecutive=consecutive,
constraint=constraint_)
annotation = Annotation()
diff = list(np.where(np.diff(converted_y))[0])
diff = [-1] + diff + [len(converted_y)]
for t, T in pairwise(diff):
segment = sliding_window.rangeToSegment(t, T - t)
annotation[segment] = converted_y[t + 1]
translation = self.label_converter_.inverse_mapping()
return annotation.translate(translation)
def apply(self, predictions, dimension=0):
"""Peak detection
Parameter
---------
predictions : SlidingWindowFeature
Predictions returned by segmentation approaches.
Returns
-------
segmentation : Timeline
Partition.
"""
if len(predictions.data.shape) == 1:
y = predictions.data
elif predictions.data.shape[1] == 1:
y = predictions.data[:, 0]
else:
y = predictions.data[:, dimension]
if self.log_scale:
y = np.exp(y)
sw = predictions.sliding_window
precision = sw.step
order = max(1, int(np.rint(self.min_duration / precision)))
indices = scipy.signal.argrelmax(y, order=order)[0]
if self.scale == 'absolute':
mini = 0
maxi = 1
elif self.scale == 'relative':
mini = np.nanmin(data)
maxi = np.nanmax(data)
elif self.scale == 'percentile':
mini = np.nanpercentile(data, 1)
maxi = np.nanpercentile(data, 99)
threshold = mini + self.alpha * (maxi - mini)
peak_time = np.array([sw[i].middle for i in indices if y[i] > threshold])
n_windows = len(y)
start_time = sw[0].start
end_time = sw[n_windows].end
boundaries = np.hstack([[start_time], peak_time, [end_time]])
segmentation = Timeline()
for i, (start, end) in enumerate(pairwise(boundaries)):
segment = Segment(start, end)
segmentation.add(segment)
return segmentation
def _partition(self, timeline, coverage):
# boundaries (as set of timestamps)
boundaries = set([])
for segment in timeline:
boundaries.add(segment.start)
boundaries.add(segment.end)
# partition (as timeline)
partition = Annotation()
for start, end in pairwise(sorted(boundaries)):
segment = Segment(start, end)
partition[segment] = '_'
return partition.crop(coverage, mode='intersection').anonymize_tracks()
def _partition(self, timeline, coverage):
# boundaries (as set of timestamps)
boundaries = set([])
for segment in timeline:
boundaries.add(segment.start)
boundaries.add(segment.end)
# partition (as timeline)
partition = Annotation()
for start, end in pairwise(sorted(boundaries)):
segment = Segment(start, end)
partition[segment] = '_'
cropped = partition.crop(coverage, mode='intersection')
return partition.crop(coverage, mode='intersection').anonymize_tracks()
def _fit_structure(self, y_iter):
K = self._n_classes()
initial = np.zeros((K,), dtype=float)
transition = np.zeros((K, K), dtype=float)
for y in y_iter:
initial[y[0]] += 1
for n, m in pairwise(y):
transition[n, m] += 1
# log-probabilities
self.initial_ = np.log(initial / np.sum(initial))
self.transition_ = np.log(transition.T / np.sum(transition, axis=1)).T
return self
def _fit_structure(self, y_iter):
K = self._n_classes()
initial = np.zeros((K, ), dtype=float)
transition = np.zeros((K, K), dtype=float)
for y in y_iter:
initial[y[0]] += 1
for n, m in pairwise(y):
transition[n, m] += 1
# log-probabilities
self.initial_ = np.log(initial / np.sum(initial))
self.transition_ = np.log(transition.T / np.sum(transition, axis=1)).T
return self
def apply(self, feature, segmentation=None):
if segmentation is None:
focus = feature.getExtent()
segmentation = Timeline(segments=[focus], uri=None)
result = Timeline()
for focus in segmentation:
x, y = list(zip(*[
(m, d) for m, d in self.iterdiff(feature, focus)
]))
x = np.array(x)
y = np.array(y)
# find local maxima
order = 1
if self.min_duration > 0:
order = int(self.min_duration / self.step)
maxima = scipy.signal.argrelmax(y, order=order)
x = x[maxima]
y = y[maxima]
# only keep high enough local maxima
high_maxima = np.where(y > self.threshold)
# create list of segment boundaries
# do not forget very first and last boundaries
boundaries = itertools.chain(
[focus.start], x[high_maxima], [focus.end]
)
# create list of segments from boundaries
segments = [Segment(*p) for p in pairwise(boundaries)]
result.update(Timeline(segments=segments))
return result
def apply(self, predictions):
"""Peak detection
Parameter
---------
predictions : SlidingWindowFeature
Predictions returned by segmentation approaches.
Returns
-------
segmentation : Timeline
Partition.
"""
y = predictions.data
sw = predictions.sliding_window
precision = sw.step
order = int(np.rint(self.min_duration / precision))
indices = scipy.signal.argrelmax(y, order=order)[0]
mini = np.nanpercentile(y, 5)
maxi = np.nanpercentile(y, 95)
threshold = mini + self.alpha * (maxi - mini)
peak_time = np.array([sw[i].middle for i in indices if y[i] > threshold])
n_windows = len(y)
start_time = sw[0].start
end_time = sw[n_windows].end
boundaries = np.hstack([[start_time], peak_time, [end_time]])
segmentation = Timeline()
for i, (start, end) in enumerate(pairwise(boundaries)):
segment = Segment(start, end)
segmentation.add(segment)
return segmentation
def apply(self, feature, segmentation=None):
if segmentation is None:
focus = feature.getExtent()
segmentation = Timeline(segments=[focus], uri=None)
result = Timeline()
for focus in segmentation:
x, y = list(
zip(*[(m, d) for m, d in self.iterdiff(feature, focus)]))
x = np.array(x)
y = np.array(y)
# find local maxima
order = 1
if self.min_duration > 0:
order = int(self.min_duration / self.step)
maxima = scipy.signal.argrelmax(y, order=order)
x = x[maxima]
y = y[maxima]
# only keep high enough local maxima
high_maxima = np.where(y > self.threshold)
# create list of segment boundaries
# do not forget very first and last boundaries
boundaries = itertools.chain([focus.start], x[high_maxima],
[focus.end])
# create list of segments from boundaries
segments = [Segment(*p) for p in pairwise(boundaries)]
result.update(Timeline(segments=segments))
return result
def loss_z(self, fX, y, n, *args):
"""Differentiable loss
Parameters
----------
fX : np.array (n_sequences, n_samples, n_dimensions)
Stacked groups of internal embeddings.
y : (batch_size, ) numpy array
Label of each group.
n : (batch_size, ) numpy array
Number of sequences per group (np.sum(n) == n_sequences)
Returns
-------
loss : float
Loss.
"""
indices = np.hstack([[0], np.cumsum(n)])
fX_sum = ag_np.stack([
ag_np.sum(ag_np.sum(fX[i:j], axis=0), axis=0)
for i, j in pairwise(indices)
])
return self.loss_y(self.l2_normalize(fX_sum), y, *args)
def validate(self,
protocol_name,
subset='development',
aggregate=False,
every=1,
start=0):
# prepare paths
validate_dir = self.VALIDATE_DIR.format(train_dir=self.train_dir_,
protocol=protocol_name)
validate_txt = self.VALIDATE_TXT.format(
validate_dir=validate_dir,
subset=subset,
aggregate='aggregate.' if aggregate else '')
validate_png = self.VALIDATE_PNG.format(
validate_dir=validate_dir,
subset=subset,
aggregate='aggregate.' if aggregate else '')
validate_eps = self.VALIDATE_EPS.format(
validate_dir=validate_dir,
subset=subset,
aggregate='aggregate.' if aggregate else '')
# create validation directory
mkdir_p(validate_dir)
# Build validation set
if aggregate:
X, n, y = self._validation_set_z(protocol_name, subset=subset)
else:
X, y = self._validation_set_y(protocol_name, subset=subset)
# list of equal error rates, and epoch to process
eers, epoch = SortedDict(), start
desc_format = ('Best EER = {best_eer:.2f}% @ epoch #{best_epoch:d} ::'
' EER = {eer:.2f}% @ epoch #{epoch:d} :')
progress_bar = tqdm(unit='epoch')
with open(validate_txt, mode='w') as fp:
# watch and evaluate forever
while True:
# last completed epochs
completed_epochs = self.get_epochs(self.train_dir_) - 1
if completed_epochs < epoch:
time.sleep(60)
continue
# if last completed epoch has already been processed
# go back to first epoch that hasn't been processed yet
process_epoch = epoch if completed_epochs in eers \
else completed_epochs
# do not validate this epoch if it has been done before...
if process_epoch == epoch and epoch in eers:
epoch += every
progress_bar.update(every)
continue
weights_h5 = LoggingCallback.WEIGHTS_H5.format(
log_dir=self.train_dir_, epoch=process_epoch)
# this is needed for corner case when training is started from
# an epoch > 0
if not isfile(weights_h5):
time.sleep(60)
continue
# sleep 5 seconds to let the checkpoint callback finish
time.sleep(5)
embedding = keras.models.load_model(
weights_h5, custom_objects=CUSTOM_OBJECTS, compile=False)
if aggregate:
def embed(X):
func = K.function([
embedding.get_layer(name='input').input,
K.learning_phase()
], [embedding.get_layer(name='internal').output])
return func([X, 0])[0]
else:
embed = embedding.predict
# embed all validation sequences
fX = embed(X)
if aggregate:
indices = np.hstack([[0], np.cumsum(n)])
fX = np.stack([
np.sum(np.sum(fX[i:j], axis=0), axis=0)
for i, j in pairwise(indices)
])
fX = l2_normalize(fX)
# compute pairwise distances
y_pred = pdist(fX, metric=self.approach_.metric)
# compute pairwise groundtruth
y_true = pdist(y, metric='chebyshev') < 1
# estimate equal error rate
_, _, _, eer = det_curve(y_true, y_pred, distances=True)
eers[process_epoch] = eer
# save equal error rate to file
fp.write(
self.VALIDATE_TXT_TEMPLATE.format(epoch=process_epoch,
eer=eer))
fp.flush()
# keep track of best epoch so far
best_epoch = eers.iloc[np.argmin(eers.values())]
best_eer = eers[best_epoch]
progress_bar.set_description(
desc_format.format(epoch=process_epoch,
eer=100 * eer,
best_epoch=best_epoch,
best_eer=100 * best_eer))
# plot
fig = plt.figure()
plt.plot(eers.keys(), eers.values(), 'b')
plt.plot([best_epoch], [best_eer], 'bo')
plt.plot([eers.iloc[0], eers.iloc[-1]], [best_eer, best_eer],
'k--')
plt.grid(True)
plt.xlabel('epoch')
plt.ylabel('EER on {subset}'.format(subset=subset))
TITLE = '{best_eer:.5g} @ epoch #{best_epoch:d}'
title = TITLE.format(best_eer=best_eer,
best_epoch=best_epoch,
subset=subset)
plt.title(title)
plt.tight_layout()
plt.savefig(validate_png, dpi=75)
plt.savefig(validate_eps)
plt.close(fig)
# go to next epoch
if epoch == process_epoch:
epoch += every
progress_bar.update(every)
else:
progress_bar.update(0)
progress_bar.close()
