def vad_metrics(predictions,
reference_segments,
sr=22050,
window_length=int(np.floor(0.032 * 22050)),
hop_length=int(np.floor(0.016 * 22050))):
frame_times = librosa.frames_to_time(range(len(predictions)),
sr=sr,
hop_length=hop_length,
n_fft=window_length)
predicted_segments = voice_segments(predictions, frame_times)
hypothesis = Annotation()
for seg in predicted_segments:
hypothesis[Segment(seg[0], seg[1])] = 1
reference = Annotation()
for seg in reference_segments:
reference[Segment(seg[0], seg[1])] = 1
precision = DetectionPrecision()(reference, hypothesis)
error = DetectionErrorRate()(reference, hypothesis)
recall = DetectionRecall()(reference, hypothesis)
accuracy = DetectionAccuracy()(reference, hypothesis)
metrics = {
"precision": precision,
"error": error,
"recall": recall,
"accuracy": accuracy
}
print(metrics)
return metrics
def get_detection_metrics(reference, hypothesis, uem=None):
metric_dict = {}
metric = DetectionErrorRate()
met = metric(reference, hypothesis, uem=uem)
metric_dict[metric.metric_name()] = met
metric = DetectionAccuracy()
met = metric(reference, hypothesis, uem=uem)
metric_dict[metric.metric_name()] = met
metric = DetectionPrecision()
met = metric(reference, hypothesis, uem=uem)
metric_dict[metric.metric_name()] = met
metric = DetectionRecall()
met = metric(reference, hypothesis, uem=uem)
metric_dict[metric.metric_name()] = met
return metric_dict
def get_metric(
self, parallel=False
) -> Union[DetectionErrorRate, DetectionPrecisionRecallFMeasure]:
"""Return new instance of detection metric"""
if self.fscore:
return DetectionPrecisionRecallFMeasure(
collar=0.0, skip_overlap=False, parallel=parallel
)
else:
return DetectionErrorRate(collar=0.0, skip_overlap=False, parallel=parallel)
def test_detailed(reference, hypothesis):
detectionErrorRate = DetectionErrorRate()
details = detectionErrorRate(reference, hypothesis, detailed=True)
rate = details['detection error rate']
npt.assert_almost_equal(rate, 0.3125, decimal=7)
false_alarm = details['false alarm']
npt.assert_almost_equal(false_alarm, 3.0, decimal=7)
missed_detection = details['miss']
npt.assert_almost_equal(missed_detection, 2.0, decimal=7)
total = details['total']
npt.assert_almost_equal(total, 16.0, decimal=7)
def detection(protocol, subset, hypotheses, collar=0.0, skip_overlap=False):
options = {
'collar': collar,
'skip_overlap': skip_overlap,
'parallel': True
}
metrics = {
'error': DetectionErrorRate(**options),
'accuracy': DetectionAccuracy(**options),
'precision': DetectionPrecision(**options),
'recall': DetectionRecall(**options)
}
reports = get_reports(protocol, subset, hypotheses, metrics)
report = metrics['error'].report(display=False)
accuracy = metrics['accuracy'].report(display=False)
precision = metrics['precision'].report(display=False)
recall = metrics['recall'].report(display=False)
report['accuracy', '%'] = accuracy[metrics['accuracy'].name, '%']
report['precision', '%'] = precision[metrics['precision'].name, '%']
report['recall', '%'] = recall[metrics['recall'].name, '%']
report = reindex(report)
columns = list(report.columns)
report = report[[columns[0]] + columns[-3:] + columns[1:-3]]
summary = 'Detection (collar = {0:g} ms{1})'.format(
1000 * collar, ', no overlap' if skip_overlap else '')
headers = [summary] + \
[report.columns[i][0] for i in range(4)] + \
['%' if c[1] == '%' else c[0] for c in report.columns[4:]]
print(
tabulate(report,
headers=headers,
tablefmt="simple",
floatfmt=".2f",
numalign="decimal",
stralign="left",
missingval="",
showindex="default",
disable_numparse=False))
def SADError(segments, AudioDataSet, annotationlist, audioLength):
reference = Annotation()
treeA = ET.parse(annotationlist[0])
rootA = treeA.getroot()
for child in rootA.findall('segment'):
start, end = float(child.get('transcriber_start')), float(
child.get('transcriber_end'))
reference[Segment(start, end)] = 'A'
treeB = ET.parse(annotationlist[1])
rootB = treeB.getroot()
for child in rootB.findall('segment'):
start, end = float(child.get('transcriber_start')), float(
child.get('transcriber_end'))
reference[Segment(start, end)] = 'A'
treeC = ET.parse(annotationlist[2])
rootC = treeC.getroot()
for child in rootC.findall('segment'):
start, end = float(child.get('transcriber_start')), float(
child.get('transcriber_end'))
reference[Segment(start, end)] = 'A'
treeD = ET.parse(annotationlist[3])
rootD = treeD.getroot()
for child in rootD.findall('segment'):
start, end = float(child.get('transcriber_start')), float(
child.get('transcriber_end'))
reference[Segment(start, end)] = 'A'
hypothesis = Annotation()
for seg in segments:
start = seg[0]
end = seg[1]
hypothesis[Segment(start, end)] = 'A'
metric = DetectionErrorRate()
uem = Timeline([Segment(0, audioLength)])
print('SAD Error Rate: %.2f %%' %
(metric(reference, hypothesis, uem=uem) * 100))
return metric, reference, hypothesis
def fun(threshold):
binarizer = Binarize(onset=threshold,
offset=threshold,
log_scale=False)
protocol = get_protocol(protocol_name, progress=False,
preprocessors=self.preprocessors_)
metric = DetectionErrorRate()
# NOTE -- embarrasingly parallel
# TODO -- parallelize this
file_generator = getattr(protocol, subset)()
for current_file in file_generator:
uri = get_unique_identifier(current_file)
hypothesis = binarizer.apply(
predictions[uri], dimension=0).to_annotation()
reference = current_file['annotation']
uem = get_annotated(current_file)
_ = metric(reference, hypothesis, uem=uem)
return abs(metric)
def main():
parser = argparse.ArgumentParser(
description='score SAD output', add_help=True)
parser.add_argument(
'ref_segments', metavar='ref-segments', type=Path,
help='path to reference segments file')
parser.add_argument(
'sys_segments', metavar='sys-segments', type=Path,
help='path to system segments file')
parser.add_argument(
'recordings_table', metavar='recordings-table', type=Path,
help='path to recordings table')
parser.add_argument(
'-u,--uem', metavar='FILE', default=None, type=Path, dest='uem_path',
help='un-partitioned evaluation map file (default: %(default)s)')
parser.add_argument(
'--collar', nargs=None, default=0.0, type=float, metavar='DUR',
help='collar size in seconds (default: %(default)s)')
parser.add_argument(
'--precision', nargs=None, default=2, type=int, metavar='DIGITS',
help='number of decimal places to print (default: %(default)s)')
parser.add_argument(
'--table-format', nargs=None, default='simple',
metavar='FMT',
help='tabulate table format (default: %(default)s)')
parser.add_argument(
'--n-jobs', nargs=None, default=1, type=int, metavar='JOBS',
help='number of parallel jobs to run (default: %(default)s)')
if len(sys.argv) == 1:
parser.print_help()
sys.exit(1)
args = parser.parse_args()
# Load annotations, scoring regions, etc.
ref_annotations = read_segments_file(args.ref_segments)
sys_annotations = read_segments_file(args.sys_segments)
annotated = {}
if args.uem_path:
annotated = read_uem_file(args.uem_path)
ref_annotations = {uri:ann for uri, ann in ref_annotations.items()
if uri in annotated}
sys_annotations = {uri:ann for uri, ann in sys_annotations.items()
if uri in annotated}
recordings = Recording.annotations_to_recordings(
ref_annotations, sys_annotations, annotated=annotated)
domains = load_domains(args.recordings_table)
# Score in parallel.
collar = 2*args.collar # Accommodation for how pyannote defines collar.
kwargs = {
'collar' : collar,
'skip_overlap' : False,
'parallel' : False}
metrics = {
'dcf' : DetectionCostFunction(
fa_weight=0.25, miss_weight=0.75, **kwargs),
'der' : DetectionErrorRate(**kwargs)}
per_domain_metrics = score_recordings(
recordings, metrics, domains=domains, n_jobs=args.n_jobs)
# Report metrics as table on STDOUT.
domain_dfs = []
for dname in per_domain_metrics:
domain_dfs.append(get_scores_dataframe(
per_domain_metrics[dname], domain_name=dname))
scores_df = pd.concat(domain_dfs)
tbl = tabulate(
scores_df, showindex=False, headers=scores_df.columns,
tablefmt=args.table_format, floatfmt=f'.{args.precision}f')
print(tbl)
def test_error_rate(reference, hypothesis):
detectionErrorRate = DetectionErrorRate()
error_rate = detectionErrorRate(reference, hypothesis)
npt.assert_almost_equal(error_rate, 0.3125, decimal=7)
def test(dataset, medium_template, config_yml, weights_h5, output_dir):
# load configuration file
with open(config_yml, 'r') as fp:
config = yaml.load(fp)
# this is where model architecture was saved
architecture_yml = os.path.dirname(
os.path.dirname(weights_h5)) + '/architecture.yml'
# -- DATASET --
db, task, protocol, subset = dataset.split('.')
database = get_database(db, medium_template=medium_template)
protocol = database.get_protocol(task, protocol)
if not hasattr(protocol, subset):
raise NotImplementedError('')
file_generator = getattr(protocol, subset)()
# -- FEATURE EXTRACTION --
# input sequence duration
duration = config['feature_extraction']['duration']
# MFCCs
feature_extractor = YaafeMFCC(**config['feature_extraction']['mfcc'])
# normalization
normalize = config['feature_extraction']['normalize']
# -- TESTING --
# overlap ratio between each window
overlap = config['testing']['overlap']
step = duration * (1. - overlap)
# prediction smoothing
onset = config['testing']['binarize']['onset']
offset = config['testing']['binarize']['offset']
binarizer = Binarize(onset=0.5, offset=0.5)
sequence_labeling = SequenceLabeling.from_disk(architecture_yml,
weights_h5)
aggregation = SequenceLabelingAggregation(sequence_labeling,
feature_extractor,
normalize=normalize,
duration=duration,
step=step)
collar = 0.500
error_rate = DetectionErrorRate(collar=collar)
accuracy = DetectionAccuracy(collar=collar)
precision = DetectionPrecision(collar=collar)
recall = DetectionRecall(collar=collar)
LINE = '{uri} {e:.3f} {a:.3f} {p:.3f} {r:.3f} {f:.3f}\n'
PATH = '{output_dir}/eval.{dataset}.{subset}.txt'
path = PATH.format(output_dir=output_dir, dataset=dataset, subset=subset)
with open(path, 'w') as fp:
header = '# uri error accuracy precision recall f_measure\n'
fp.write(header)
fp.flush()
for current_file in file_generator:
uri = current_file['uri']
wav = current_file['medium']['wav']
annotated = current_file['annotated']
annotation = current_file['annotation']
predictions = aggregation.apply(wav)
hypothesis = binarizer.apply(predictions, dimension=1)
e = error_rate(annotation, hypothesis, uem=annotated)
a = accuracy(annotation, hypothesis, uem=annotated)
p = precision(annotation, hypothesis, uem=annotated)
r = recall(annotation, hypothesis, uem=annotated)
f = f_measure(p, r)
line = LINE.format(uri=uri, e=e, a=a, p=p, r=r, f=f)
fp.write(line)
fp.flush()
PATH = '{output_dir}/{uri}.json'
path = PATH.format(output_dir=output_dir, uri=uri)
dump_to(hypothesis, path)
# average on whole corpus
uri = '{dataset}.{subset}'.format(dataset=dataset, subset=subset)
e = abs(error_rate)
a = abs(accuracy)
p = abs(precision)
r = abs(recall)
f = f_measure(p, r)
line = LINE.format(uri=uri, e=e, a=a, p=p, r=r, f=f)
fp.write(line)
fp.flush()
def get_tune_metric(self):
return DetectionErrorRate()
def get_metric(self) -> Union[DetectionErrorRate, DiarizationErrorRate]:
if self.only_sad:
return DetectionErrorRate(collar=0.0)
else:
return DiarizationErrorRate(collar=0.0, skip_overlap=False)
def validate_epoch(self, epoch, protocol_name, subset='development',
validation_data=None):
# load model for current epoch
model = self.load_model(epoch).to(self.device)
model.eval()
if isinstance(self.feature_extraction_, Precomputed):
self.feature_extraction_.use_memmap = False
duration = self.task_.duration
step = .25 * duration
sequence_labeling = SequenceLabeling(
model, self.feature_extraction_, duration=duration,
step=.25 * duration, batch_size=self.batch_size,
source='audio', device=self.device)
protocol = get_protocol(protocol_name, progress=False,
preprocessors=self.preprocessors_)
metric = DetectionErrorRate()
predictions = {}
file_generator = getattr(protocol, subset)()
for current_file in file_generator:
uri = get_unique_identifier(current_file)
scores = sequence_labeling.apply(current_file)
if model.logsoftmax:
scores = SlidingWindowFeature(
1. - np.exp(scores.data[:, 0]),
scores.sliding_window)
else:
scores = SlidingWindowFeature(
1. - scores.data[:, 0],
scores.sliding_window)
predictions[uri] = scores
def fun(threshold):
binarizer = Binarize(onset=threshold,
offset=threshold,
log_scale=False)
protocol = get_protocol(protocol_name, progress=False,
preprocessors=self.preprocessors_)
metric = DetectionErrorRate()
# NOTE -- embarrasingly parallel
# TODO -- parallelize this
file_generator = getattr(protocol, subset)()
for current_file in file_generator:
uri = get_unique_identifier(current_file)
hypothesis = binarizer.apply(
predictions[uri], dimension=0).to_annotation()
reference = current_file['annotation']
uem = get_annotated(current_file)
_ = metric(reference, hypothesis, uem=uem)
return abs(metric)
res = scipy.optimize.minimize_scalar(
fun, bounds=(0., 1.), method='bounded', options={'maxiter': 10})
return {
'speech_activity_detection/error': {'minimize': True,
'value': res.fun},
'speech_activity_detection/threshold': {'minimize': 'NA',
'value': res.x}}
