def from_file(self, current_file): from_annotation = current_file['annotation'] if self.source == 'annotated': support = get_annotated(current_file) elif self.source == 'support': support = current_file['annotation'].get_timeline().support() elif self.source == 'annotation': support = current_file['annotation'] elif self.source == 'audio': from pyannote.audio.features.utils import get_audio_duration support = get_audio_duration(current_file) else: raise ValueError( 'source must be one of "annotated", "annotation", "support" ' 'or "audio"') if self.heterogeneous: generator = self.iter_heterogeneous_segments(from_annotation, support) else: generator = self.iter_segments(from_annotation) for segment, label in generator: if label is None and self.skip_unlabeled: continue yield segment, label
def process_one(item, hypotheses=None, metrics=None): reference = item['annotation'] hypothesis = get_hypothesis(hypotheses, item) uem = get_annotated(item) return { key: metric(reference, hypothesis, uem=uem) for key, metric in metrics.items() }
def preprocess(self, current_file, identifier=None): """Pre-compute file-wise X and y""" if not hasattr(self, 'preprocessed_'): self.preprocessed_ = {} self.preprocessed_['X'] = {} self.preprocessed_['y'] = {} self.preprocessed_['X'][identifier] = current_file['features'] # if labels have already been extracted, do nothing if identifier in self.preprocessed_.setdefault('y', {}): return current_file # get features as pyannote.core.SlidingWindowFeature instance X = self.preprocessed_['X'][identifier] sw = X.sliding_window n_samples = X.getNumber() #self.shape = (n_samples, dimension) annotated = get_annotated(current_file) annotation = current_file['annotation'] prediction = current_file['prediction'] if not hasattr(self, 'input_shape'): self.input_shape = (sw.samples(self.duration, mode='center'), X.data.shape[1]) if self.source == 'annotation': n_classes = len(prediction.labels()) self.n_classes = n_classes y = np.zeros((n_samples + 4, n_classes), dtype=np.int8) label_map = { label: idx for idx, label in enumerate(prediction.labels()) } else: n_classes = len(prediction.labels()) + 1 self.n_classes = n_classes y = np.zeros((n_samples + 4, n_classes), dtype=np.int8) label_map = { label: idx + 1 for idx, label in enumerate(prediction.labels()) } for segment, _, label in prediction.itertracks(label=True): indices = sw.crop(segment, mode='loose') y[indices, label_map[label]] = 1 y = SlidingWindowFeature(y[:-1], sw) self.preprocessed_['y'][identifier] = y return current_file
def from_file(self, current_file): if isinstance(self.source, (Segment, Timeline)): source = self.source elif self.source == 'annotated': source = get_annotated(current_file) elif self.source == 'annotated_extent': source = get_annotated(current_file).extent() elif self.source == 'annotation': source = current_file['annotation'] elif self.source == 'support': source = current_file['annotation'].get_timeline().support() elif self.source == 'audio': from pyannote.audio.features.utils import get_audio_duration source = get_audio_duration(current_file) for segment in self.iter_segments(source): yield segment
def preprocess(self, current_file, identifier=None): """Pre-compute file-wise X and y""" # extract features for the whole file # (if it has not been done already) current_file = self.periodic_preprocess(current_file, identifier=identifier) # if labels have already been extracted, do nothing if identifier in self.preprocessed_.setdefault('y', {}): return current_file # get features as pyannote.core.SlidingWindowFeature instance X = self.preprocessed_['X'][identifier] sw = X.sliding_window n_samples = X.getNumber() y = np.zeros((n_samples + 4, 1), dtype=np.int8) - 1 # [-1] ==> unknown / [0] ==> not change part / [1] ==> change part annotated = get_annotated(current_file) annotation = current_file['annotation'] segments = [] for segment, _ in annotation.itertracks(): segments.append( Segment(segment.start - self.balance, segment.start + self.balance)) segments.append( Segment(segment.end - self.balance, segment.end + self.balance)) change_part = Timeline(segments).support().crop(annotated, mode='intersection') # iterate over non-change regions for non_changes in change_part.gaps(annotated): indices = sw.crop(non_changes, mode='loose') y[indices, 0] = 0 # iterate over change regions for changes in change_part: indices = sw.crop(changes, mode='loose') y[indices, 0] = 1 y = SlidingWindowFeature(y[:-1], sw) self.preprocessed_['y'][identifier] = y return current_file
def helper_cluster_tune(item_segmentation_features, metric=None, covariance_type='full', penalty_coef=3.5, **kwargs): """Apply clustering on prediction and evaluate the result Parameters ---------- item : dict Protocol item. segmentation : Annotation Initial segmentation features : SlidingWindowFeature Precomputed features metric : BaseMetric, optional covariance_type : {'diag', 'full'}, optional penalty_coef : float, optional **kwargs : dict Passed to clustering.apply() Returns ------- value : float Metric value """ item, segmentation, features = item_segmentation_features clustering = pyannote.algorithms.clustering.bic.BICClustering( covariance_type=covariance_type, penalty_coef=penalty_coef) uem = get_annotated(item) reference = item['annotation'] hypothesis = clustering(segmentation, features=features) result = metric(reference, hypothesis, uem=uem) return result
def helper_binarize_tune(item_prediction, binarizer=None, metric=None, **kwargs): """Apply binarizer on prediction and evaluate the result Parameters ---------- item : dict Protocol item. prediction : SlidingWindowFeature binarizer : Binarize, optional metric : BaseMetric, optional **kwargs : dict Passed to binarizer.apply() Returns ------- value : float Metric value """ item, prediction = item_prediction uem = get_annotated(item) reference = item['annotation'] hypothesis_timeline = binarizer.apply(prediction, **kwargs) # convert from timeline to annnotation hypothesis = Annotation(uri=hypothesis_timeline.uri) for s, segment in enumerate(hypothesis_timeline): hypothesis[segment] = s result = metric(reference, hypothesis, uem=uem) return result
skip_overlap=True) metric3 = GreedyDiarizationErrorRate(parallel=False, collar=0.500, skip_overlap=False) from optimize_cluster import speaker_diarization from pyannote.audio.features import Precomputed file_list = [] for current_file in getattr(protocol, subset)(): uri = get_unique_identifier(current_file).split('/')[1] hypothesis = diarization_res[uri] reference = current_file['annotation'] current_file['prediction'] = hypothesis file_list.append(current_file) uem = get_annotated(current_file) metric1(reference, hypothesis, uem=uem) metric2(reference, hypothesis, uem=uem) metric3(reference, hypothesis, uem=uem) print(abs(metric1)) print(abs(metric2)) print(abs(metric3)) config_yml = arguments['<config_yml>'] models_dir = arguments['<models_dir>'] num_epoch = int(arguments['<num_epoch>']) embedding_precomputed = '/vol/work1/yin/embedding/20180124' feature_precomputed = arguments['<feature_pre>']
def validate(self, protocol_name, subset='development'): # prepare paths validate_dir = self.VALIDATE_DIR.format(train_dir=self.train_dir_, protocol=protocol_name) validate_txt = self.VALIDATE_TXT.format(validate_dir=validate_dir, subset=subset) validate_png = self.VALIDATE_PNG.format(validate_dir=validate_dir, subset=subset) validate_eps = self.VALIDATE_EPS.format(validate_dir=validate_dir, subset=subset) # create validation directory mkdir_p(validate_dir) # Build validation set y = self._validation_set(protocol_name, subset=subset) # list of equal error rates, and current epoch eers, epoch = [], 0 desc_format = ('EER = {eer:.2f}% @ epoch #{epoch:d} ::' ' Best EER = {best_eer:.2f}% @ epoch #{best_epoch:d} :') progress_bar = tqdm(unit='epoch', total=1000) with open(validate_txt, mode='w') as fp: # watch and evaluate forever while True: weights_h5 = LoggingCallback.WEIGHTS_H5.format( log_dir=self.train_dir_, epoch=epoch) # wait until weight file is available if not isfile(weights_h5): time.sleep(60) continue # load model for current epoch sequence_labeling = SequenceLabeling.from_disk( self.train_dir_, epoch) # initialize sequence labeling duration = self.config_['sequences']['duration'] step = duration # hack to make things faster # step = self.config_['sequences']['step'] aggregation = SequenceLabelingAggregation( sequence_labeling, self.feature_extraction_, duration=duration, step=step) aggregation.cache_preprocessed_ = False # estimate equal error rate (average of all files) eers_ = [] protocol = get_protocol(protocol_name, progress=False, preprocessors=self.preprocessors_) file_generator = getattr(protocol, subset)() for current_file in file_generator: identifier = get_unique_identifier(current_file) uem = get_annotated(current_file) y_true = y[identifier].crop(uem)[:, 1] counts = Counter(y_true) if counts[0] * counts[1] == 0: continue y_pred = aggregation.apply(current_file).crop(uem)[:, 1] _, _, _, eer = det_curve(y_true, y_pred, distances=False) eers_.append(eer) eer = np.mean(eers_) eers.append(eer) # save equal error rate to file fp.write( self.VALIDATE_TXT_TEMPLATE.format(epoch=epoch, eer=eer)) fp.flush() # keep track of best epoch so far best_epoch, best_eer = np.argmin(eers), np.min(eers) progress_bar.set_description( desc_format.format(epoch=epoch, eer=100 * eer, best_epoch=best_epoch, best_eer=100 * best_eer)) progress_bar.update(1) # plot fig = plt.figure() plt.plot(eers, 'b') plt.plot([best_epoch], [best_eer], 'bo') plt.plot([0, epoch], [best_eer, best_eer], 'k--') plt.grid(True) plt.xlabel('epoch') plt.ylabel('EER on {subset}'.format(subset=subset)) TITLE = '{best_eer:.5g} @ epoch #{best_epoch:d}' title = TITLE.format(best_eer=best_eer, best_epoch=best_epoch, subset=subset) plt.title(title) plt.tight_layout() plt.savefig(validate_png, dpi=75) plt.savefig(validate_eps) plt.close(fig) # validate next epoch epoch += 1 progress_bar.close()