def get_mapping(reference, system): """ get speaker mapping between system and reference""" metric = DiarizationErrorRate() mapping = metric.optimal_mapping(reference, system) return mapping
def score_labels(AUDIO_RTTM_MAP, all_reference, all_hypothesis, collar=0.25, ignore_overlap=True): """ calculates DER, CER, FA and MISS Args: AUDIO_RTTM_MAP : Dictionary containing information provided from manifestpath all_reference (list[uniq_name,Annotation]): reference annotations for score calculation all_hypothesis (list[uniq_name,Annotation]): hypothesis annotations for score calculation Returns: metric (pyannote.DiarizationErrorRate): Pyannote Diarization Error Rate metric object. This object contains detailed scores of each audiofile. mapping (dict): Mapping dict containing the mapping speaker label for each audio input < Caveat > Unlike md-eval.pl, "no score" collar in pyannote.metrics is the maximum length of "no score" collar from left to right. Therefore, if 0.25s is applied for "no score" collar in md-eval.pl, 0.5s should be applied for pyannote.metrics. """ metric = None if len(all_reference) == len(all_hypothesis): metric = DiarizationErrorRate(collar=2 * collar, skip_overlap=ignore_overlap) mapping_dict = {} for (reference, hypothesis) in zip(all_reference, all_hypothesis): ref_key, ref_labels = reference _, hyp_labels = hypothesis uem = AUDIO_RTTM_MAP[ref_key].get('uem_filepath', None) if uem is not None: uem = uem_timeline_from_file(uem_file=uem, uniq_name=ref_key) metric(ref_labels, hyp_labels, uem=uem, detailed=True) mapping_dict[ref_key] = metric.optimal_mapping( ref_labels, hyp_labels) DER = abs(metric) CER = metric['confusion'] / metric['total'] FA = metric['false alarm'] / metric['total'] MISS = metric['missed detection'] / metric['total'] logging.info( "Cumulative Results for collar {} sec and ignore_overlap {}: \n FA: {:.4f}\t MISS {:.4f}\t \ Diarization ER: {:.4f}\t, Confusion ER:{:.4f}".format( collar, ignore_overlap, FA, MISS, DER, CER)) return metric, mapping_dict else: logging.warning( "check if each ground truth RTTMs were present in provided manifest file. Skipping calculation of Diariazation Error Rate" ) return None
def gecko(args): hypotheses_path = args['<hypotheses_path>'] uri = args['<uri>'] colors = get_colors(uri) distances = {} if Path(hypotheses_path).exists(): hypotheses = load_rttm(hypotheses_path) hypothesis = hypotheses[uri] else: # protocol protocol = get_protocol(args['<hypotheses_path>']) reference = get_file(protocol, uri) hypothesis = reference['annotation'] annotated = get_annotated(reference) hypotheses_path = Path(hypotheses_path) protocol = args['--database.task.protocol'] features = None if protocol: protocol = get_protocol(protocol) embeddings = args['--embeddings'] reference, features = get_file(protocol, uri, embeddings=embeddings) if args['--map']: print(f"mapping {uri} with {protocol}") diarizationErrorRate = DiarizationErrorRate() annotated = get_annotated(reference) optimal_mapping = diarizationErrorRate.optimal_mapping( reference['annotation'], hypothesis, annotated) hypothesis = hypothesis.rename_labels(mapping=optimal_mapping) hypothesis = update_labels(hypothesis, distances) # tag unsure clusters distances_per_speaker = get_distances_per_speaker( features, hypothesis) if features else {} if args['--tag_na']: whole_file = Segment(0., annotated.segments_boundaries_[-1]) not_annotated = annotated.gaps(whole_file).to_annotation(na()) hypothesis = hypothesis.crop(annotated).update(not_annotated) gecko_json = annotation_to_GeckoJSON(hypothesis, distances_per_speaker, colors) if hypotheses_path.exists(): dir_path = hypotheses_path.parent else: dir_path = Path(".") json_path = os.path.join(dir_path, f'{uri}.json') with open(json_path, 'w') as file: json.dump(gecko_json, file) print(f"succefully dumped {json_path}")
def get_DER(all_reference, all_hypothesis, collar=0.5, skip_overlap=True): """ calculates DER, CER, FA and MISS Args: all_reference (list[Annotation]): reference annotations for score calculation all_hypothesis (list[Annotation]): hypothesis annotations for score calculation Returns: DER (float): Diarization Error Rate CER (float): Confusion Error Rate FA (float): False Alarm Miss (float): Miss Detection < Caveat > Unlike md-eval.pl, "no score" collar in pyannote.metrics is the maximum length of "no score" collar from left to right. Therefore, if 0.25s is applied for "no score" collar in md-eval.pl, 0.5s should be applied for pyannote.metrics. """ metric = DiarizationErrorRate(collar=collar, skip_overlap=skip_overlap, uem=None) mapping_dict = {} for k, (reference, hypothesis) in enumerate(zip(all_reference, all_hypothesis)): metric(reference, hypothesis, detailed=True) mapping_dict[k] = metric.optimal_mapping(reference, hypothesis) DER = abs(metric) CER = metric['confusion'] / metric['total'] FA = metric['false alarm'] / metric['total'] MISS = metric['missed detection'] / metric['total'] metric.reset() return DER, CER, FA, MISS, mapping_dict
def test_optimal_mapping(reference, hypothesis): diarizationErrorRate = DiarizationErrorRate() mapping = diarizationErrorRate.optimal_mapping(reference, hypothesis) assert mapping == {'a': 'A', 'b': 'B', 'c': 'C'}
def test_optimal_mapping(reference, hypothesis): diarizationErrorRate = DiarizationErrorRate() mapping = diarizationErrorRate.optimal_mapping(reference, hypothesis) assert mapping == {'a': 'A', 'b': 'B', 'c': 'C'}