def scenes(self, threads): g = nx.Graph() # connect adjacent shots for shot1, shot2 in pairwise(threads.itertracks()): g.add_edge(shot1, shot2) # connect threaded shots for label in threads.labels(): for shot1, shot2 in pairwise(threads.subset([label]).itertracks()): g.add_edge(shot1, shot2) scenes = threads.copy() # group all shots of intertwined threads for shots in sorted(sorted(bc) for bc in nx.biconnected_components(g)): if len(shots) < 3: continue common_label = scenes[shots[0]] for shot in shots: scenes[shot] = common_label return scenes
def predict(self, features, min_duration=None, constraint=None): """ Parameters ---------- min_duration : float or dict, optional Minimum duration for each label, in seconds. """ constraint_ = self._constraint(constraint, features) consecutive = self._consecutive(min_duration, features) X = self.X(features, unknown="keep") sliding_window = features.sliding_window converted_y = self.classifier_.predict(X, consecutive=consecutive, constraint=constraint_) annotation = Annotation() diff = list(np.where(np.diff(converted_y))[0]) diff = [-1] + diff + [len(converted_y)] for t, T in pairwise(diff): segment = sliding_window.rangeToSegment(t, T - t) annotation[segment] = converted_y[t + 1] translation = self.label_converter_.inverse_mapping() return annotation.translate(translation)
def apply(self, features, segmentation=None): """ Parameters ---------- features : Features segmentation : Timeline, optional """ if segmentation is None: segmentation = Timeline(segments=[features.getExtent()]) sliding_window = features.sliding_window min_samples = sliding_window.durationToSamples(self.min_duration) precision = sliding_window.durationToSamples(self.precision) segmenter = SKLearnBICSegmentation( penalty_coef=self.penalty_coef, covariance_type=self.covariance_type, min_samples=min_samples, precision=precision) result = Timeline() for long_segment in segmentation: X = features.crop(long_segment) boundaries = segmenter.apply(X) for t, T in pairwise(boundaries): segment = sliding_window.rangeToSegment(t, T - t) shifted_segment = Segment(long_segment.start + segment.start, long_segment.start + segment.end) result.add(shifted_segment) return result
def predict(self, features, min_duration=None, constraint=None): """ Parameters ---------- min_duration : float or dict, optional Minimum duration for each label, in seconds. """ constraint_ = self._constraint(constraint, features) consecutive = self._consecutive(min_duration, features) X = self.X(features, unknown='keep') sliding_window = features.sliding_window converted_y = self.classifier_.predict(X, consecutive=consecutive, constraint=constraint_) annotation = Annotation() diff = list(np.where(np.diff(converted_y))[0]) diff = [-1] + diff + [len(converted_y)] for t, T in pairwise(diff): segment = sliding_window.rangeToSegment(t, T - t) annotation[segment] = converted_y[t + 1] translation = self.label_converter_.inverse_mapping() return annotation.translate(translation)
def apply(self, predictions, dimension=0): """Peak detection Parameter --------- predictions : SlidingWindowFeature Predictions returned by segmentation approaches. Returns ------- segmentation : Timeline Partition. """ if len(predictions.data.shape) == 1: y = predictions.data elif predictions.data.shape[1] == 1: y = predictions.data[:, 0] else: y = predictions.data[:, dimension] if self.log_scale: y = np.exp(y) sw = predictions.sliding_window precision = sw.step order = max(1, int(np.rint(self.min_duration / precision))) indices = scipy.signal.argrelmax(y, order=order)[0] if self.scale == 'absolute': mini = 0 maxi = 1 elif self.scale == 'relative': mini = np.nanmin(data) maxi = np.nanmax(data) elif self.scale == 'percentile': mini = np.nanpercentile(data, 1) maxi = np.nanpercentile(data, 99) threshold = mini + self.alpha * (maxi - mini) peak_time = np.array([sw[i].middle for i in indices if y[i] > threshold]) n_windows = len(y) start_time = sw[0].start end_time = sw[n_windows].end boundaries = np.hstack([[start_time], peak_time, [end_time]]) segmentation = Timeline() for i, (start, end) in enumerate(pairwise(boundaries)): segment = Segment(start, end) segmentation.add(segment) return segmentation
def _partition(self, timeline, coverage): # boundaries (as set of timestamps) boundaries = set([]) for segment in timeline: boundaries.add(segment.start) boundaries.add(segment.end) # partition (as timeline) partition = Annotation() for start, end in pairwise(sorted(boundaries)): segment = Segment(start, end) partition[segment] = '_' return partition.crop(coverage, mode='intersection').anonymize_tracks()
def _partition(self, timeline, coverage): # boundaries (as set of timestamps) boundaries = set([]) for segment in timeline: boundaries.add(segment.start) boundaries.add(segment.end) # partition (as timeline) partition = Annotation() for start, end in pairwise(sorted(boundaries)): segment = Segment(start, end) partition[segment] = '_' cropped = partition.crop(coverage, mode='intersection') return partition.crop(coverage, mode='intersection').anonymize_tracks()
def _fit_structure(self, y_iter): K = self._n_classes() initial = np.zeros((K,), dtype=float) transition = np.zeros((K, K), dtype=float) for y in y_iter: initial[y[0]] += 1 for n, m in pairwise(y): transition[n, m] += 1 # log-probabilities self.initial_ = np.log(initial / np.sum(initial)) self.transition_ = np.log(transition.T / np.sum(transition, axis=1)).T return self
def _fit_structure(self, y_iter): K = self._n_classes() initial = np.zeros((K, ), dtype=float) transition = np.zeros((K, K), dtype=float) for y in y_iter: initial[y[0]] += 1 for n, m in pairwise(y): transition[n, m] += 1 # log-probabilities self.initial_ = np.log(initial / np.sum(initial)) self.transition_ = np.log(transition.T / np.sum(transition, axis=1)).T return self
def apply(self, feature, segmentation=None): if segmentation is None: focus = feature.getExtent() segmentation = Timeline(segments=[focus], uri=None) result = Timeline() for focus in segmentation: x, y = list(zip(*[ (m, d) for m, d in self.iterdiff(feature, focus) ])) x = np.array(x) y = np.array(y) # find local maxima order = 1 if self.min_duration > 0: order = int(self.min_duration / self.step) maxima = scipy.signal.argrelmax(y, order=order) x = x[maxima] y = y[maxima] # only keep high enough local maxima high_maxima = np.where(y > self.threshold) # create list of segment boundaries # do not forget very first and last boundaries boundaries = itertools.chain( [focus.start], x[high_maxima], [focus.end] ) # create list of segments from boundaries segments = [Segment(*p) for p in pairwise(boundaries)] result.update(Timeline(segments=segments)) return result
def apply(self, predictions): """Peak detection Parameter --------- predictions : SlidingWindowFeature Predictions returned by segmentation approaches. Returns ------- segmentation : Timeline Partition. """ y = predictions.data sw = predictions.sliding_window precision = sw.step order = int(np.rint(self.min_duration / precision)) indices = scipy.signal.argrelmax(y, order=order)[0] mini = np.nanpercentile(y, 5) maxi = np.nanpercentile(y, 95) threshold = mini + self.alpha * (maxi - mini) peak_time = np.array([sw[i].middle for i in indices if y[i] > threshold]) n_windows = len(y) start_time = sw[0].start end_time = sw[n_windows].end boundaries = np.hstack([[start_time], peak_time, [end_time]]) segmentation = Timeline() for i, (start, end) in enumerate(pairwise(boundaries)): segment = Segment(start, end) segmentation.add(segment) return segmentation
def apply(self, feature, segmentation=None): if segmentation is None: focus = feature.getExtent() segmentation = Timeline(segments=[focus], uri=None) result = Timeline() for focus in segmentation: x, y = list( zip(*[(m, d) for m, d in self.iterdiff(feature, focus)])) x = np.array(x) y = np.array(y) # find local maxima order = 1 if self.min_duration > 0: order = int(self.min_duration / self.step) maxima = scipy.signal.argrelmax(y, order=order) x = x[maxima] y = y[maxima] # only keep high enough local maxima high_maxima = np.where(y > self.threshold) # create list of segment boundaries # do not forget very first and last boundaries boundaries = itertools.chain([focus.start], x[high_maxima], [focus.end]) # create list of segments from boundaries segments = [Segment(*p) for p in pairwise(boundaries)] result.update(Timeline(segments=segments)) return result
def loss_z(self, fX, y, n, *args): """Differentiable loss Parameters ---------- fX : np.array (n_sequences, n_samples, n_dimensions) Stacked groups of internal embeddings. y : (batch_size, ) numpy array Label of each group. n : (batch_size, ) numpy array Number of sequences per group (np.sum(n) == n_sequences) Returns ------- loss : float Loss. """ indices = np.hstack([[0], np.cumsum(n)]) fX_sum = ag_np.stack([ ag_np.sum(ag_np.sum(fX[i:j], axis=0), axis=0) for i, j in pairwise(indices) ]) return self.loss_y(self.l2_normalize(fX_sum), y, *args)
def validate(self, protocol_name, subset='development', aggregate=False, every=1, start=0): # prepare paths validate_dir = self.VALIDATE_DIR.format(train_dir=self.train_dir_, protocol=protocol_name) validate_txt = self.VALIDATE_TXT.format( validate_dir=validate_dir, subset=subset, aggregate='aggregate.' if aggregate else '') validate_png = self.VALIDATE_PNG.format( validate_dir=validate_dir, subset=subset, aggregate='aggregate.' if aggregate else '') validate_eps = self.VALIDATE_EPS.format( validate_dir=validate_dir, subset=subset, aggregate='aggregate.' if aggregate else '') # create validation directory mkdir_p(validate_dir) # Build validation set if aggregate: X, n, y = self._validation_set_z(protocol_name, subset=subset) else: X, y = self._validation_set_y(protocol_name, subset=subset) # list of equal error rates, and epoch to process eers, epoch = SortedDict(), start desc_format = ('Best EER = {best_eer:.2f}% @ epoch #{best_epoch:d} ::' ' EER = {eer:.2f}% @ epoch #{epoch:d} :') progress_bar = tqdm(unit='epoch') with open(validate_txt, mode='w') as fp: # watch and evaluate forever while True: # last completed epochs completed_epochs = self.get_epochs(self.train_dir_) - 1 if completed_epochs < epoch: time.sleep(60) continue # if last completed epoch has already been processed # go back to first epoch that hasn't been processed yet process_epoch = epoch if completed_epochs in eers \ else completed_epochs # do not validate this epoch if it has been done before... if process_epoch == epoch and epoch in eers: epoch += every progress_bar.update(every) continue weights_h5 = LoggingCallback.WEIGHTS_H5.format( log_dir=self.train_dir_, epoch=process_epoch) # this is needed for corner case when training is started from # an epoch > 0 if not isfile(weights_h5): time.sleep(60) continue # sleep 5 seconds to let the checkpoint callback finish time.sleep(5) embedding = keras.models.load_model( weights_h5, custom_objects=CUSTOM_OBJECTS, compile=False) if aggregate: def embed(X): func = K.function([ embedding.get_layer(name='input').input, K.learning_phase() ], [embedding.get_layer(name='internal').output]) return func([X, 0])[0] else: embed = embedding.predict # embed all validation sequences fX = embed(X) if aggregate: indices = np.hstack([[0], np.cumsum(n)]) fX = np.stack([ np.sum(np.sum(fX[i:j], axis=0), axis=0) for i, j in pairwise(indices) ]) fX = l2_normalize(fX) # compute pairwise distances y_pred = pdist(fX, metric=self.approach_.metric) # compute pairwise groundtruth y_true = pdist(y, metric='chebyshev') < 1 # estimate equal error rate _, _, _, eer = det_curve(y_true, y_pred, distances=True) eers[process_epoch] = eer # save equal error rate to file fp.write( self.VALIDATE_TXT_TEMPLATE.format(epoch=process_epoch, eer=eer)) fp.flush() # keep track of best epoch so far best_epoch = eers.iloc[np.argmin(eers.values())] best_eer = eers[best_epoch] progress_bar.set_description( desc_format.format(epoch=process_epoch, eer=100 * eer, best_epoch=best_epoch, best_eer=100 * best_eer)) # plot fig = plt.figure() plt.plot(eers.keys(), eers.values(), 'b') plt.plot([best_epoch], [best_eer], 'bo') plt.plot([eers.iloc[0], eers.iloc[-1]], [best_eer, best_eer], 'k--') plt.grid(True) plt.xlabel('epoch') plt.ylabel('EER on {subset}'.format(subset=subset)) TITLE = '{best_eer:.5g} @ epoch #{best_epoch:d}' title = TITLE.format(best_eer=best_eer, best_epoch=best_epoch, subset=subset) plt.title(title) plt.tight_layout() plt.savefig(validate_png, dpi=75) plt.savefig(validate_eps) plt.close(fig) # go to next epoch if epoch == process_epoch: epoch += every progress_bar.update(every) else: progress_bar.update(0) progress_bar.close()