def segmentation(protocol, subset, hypotheses, tolerance=0.5):
options = {'tolerance': tolerance, 'parallel': True}
metrics = {'coverage': SegmentationCoverage(**options),
'purity': SegmentationPurity(**options),
'precision': SegmentationPrecision(**options),
'recall': SegmentationRecall(**options)}
reports = get_reports(protocol, subset, hypotheses, metrics)
coverage = metrics['coverage'].report(display=False)
purity = metrics['purity'].report(display=False)
precision = metrics['precision'].report(display=False)
recall = metrics['recall'].report(display=False)
coverage = coverage[metrics['coverage'].name]
purity = purity[metrics['purity'].name]
precision = precision[metrics['precision'].name]
recall = recall[metrics['recall'].name]
report = pd.concat([coverage, purity, precision, recall], axis=1)
report = reindex(report)
headers = ['Segmentation (tolerance = {0:g} ms)'.format(1000*tolerance),
'coverage', 'purity', 'precision', 'recall']
print(tabulate(report, headers=headers, tablefmt="simple",
floatfmt=".2f", numalign="decimal", stralign="left",
missingval="", showindex="default", disable_numparse=False))
def segmentation(protocol, subset, hypotheses, tolerance=0.5):
options = {"tolerance": tolerance, "parallel": True}
metrics = {
"coverage": SegmentationCoverage(**options),
"purity": SegmentationPurity(**options),
"precision": SegmentationPrecision(**options),
"recall": SegmentationRecall(**options),
}
reports = get_reports(protocol, subset, hypotheses, metrics)
coverage = metrics["coverage"].report(display=False)
purity = metrics["purity"].report(display=False)
precision = metrics["precision"].report(display=False)
recall = metrics["recall"].report(display=False)
coverage = coverage[metrics["coverage"].name]
purity = purity[metrics["purity"].name]
precision = precision[metrics["precision"].name]
recall = recall[metrics["recall"].name]
report = pd.concat([coverage, purity, precision, recall], axis=1)
report = reindex(report)
headers = [
"Segmentation (tolerance = {0:g} ms)".format(1000 * tolerance),
"coverage",
"purity",
"precision",
"recall",
]
print(
tabulate(
report,
headers=headers,
tablefmt="simple",
floatfmt=".2f",
numalign="decimal",
stralign="left",
missingval="",
showindex="default",
disable_numparse=False,
))
def objective_function(parameters, beta=1.0):
epoch, alpha = parameters
weights_h5 = WEIGHTS_H5.format(epoch=epoch)
sequence_embedding = SequenceEmbedding.from_disk(
architecture_yml, weights_h5)
segmentation = Segmentation(
sequence_embedding, feature_extraction,
duration=duration, step=0.100)
if epoch not in predictions:
predictions[epoch] = {}
purity = SegmentationPurity()
coverage = SegmentationCoverage()
f, n = 0., 0
for dev_file in getattr(protocol, subset)():
uri = get_unique_identifier(dev_file)
reference = dev_file['annotation']
n += 1
if uri in predictions[epoch]:
prediction = predictions[epoch][uri]
else:
prediction = segmentation.apply(dev_file)
predictions[epoch][uri] = prediction
peak = Peak(alpha=alpha)
hypothesis = peak.apply(prediction)
p = purity(reference, hypothesis)
c = coverage(reference, hypothesis)
f += f_measure(c, p, beta=beta)
return 1 - (f / n)
def seg_metrics(predictions,
reference_segments,
sr=22050,
window_length=int(np.floor(0.032 * 22050)),
hop_length=int(np.floor(0.016 * 22050))):
# frame_times = librosa.frames_to_time(range(len(predictions)), sr=sr, hop_length=hop_length, n_fft=window_length)
# predicted_segments = voice_segments(predictions, frame_times)
hypothesis = Annotation()
for seg in predictions:
hypothesis[Segment(seg[0], seg[1])] = 1
reference = Annotation()
for seg in reference_segments:
reference[Segment(seg[0], seg[1])] = 1
coverage = SegmentationCoverage()(reference, hypothesis)
purity = SegmentationPurity()(reference, hypothesis)
metrics = {"coverage": coverage, "purity": purity}
print(metrics)
return metrics
def get_segmentation_metrics(reference, hypothesis, uem=None):
metric_dict = {}
metric = SegmentationCoverage()
met = metric(reference, hypothesis, uem=uem)
metric_dict[metric.metric_name()] = met
metric = SegmentationRecall()
met = metric(reference, hypothesis, uem=uem)
metric_dict[metric.metric_name()] = met
metric = SegmentationPrecision()
met = metric(reference, hypothesis, uem=uem)
metric_dict[metric.metric_name()] = met
metric = SegmentationPurity()
met = metric(reference, hypothesis, uem=uem)
metric_dict[metric.metric_name()] = met
metric = SegmentationPurityCoverageFMeasure()
met = metric(reference, hypothesis, uem=uem)
metric_dict[metric.metric_name()] = met
return metric_dict
def test(custom=True, prefix=''):
der = DiarizationErrorRate(collar=0.5)
prec = DetectionPrecision(collar=0.5)
recall = DetectionRecall(collar=0.5)
coverage = SegmentationCoverage()
purity = SegmentationPurity()
result = {}
if os.path.exists('results.json'):
with open('results.json') as json_file:
result = json.load(json_file)
base_test = prefix + 'audio/'
test_files = os.listdir(base_test)
test_path = base_test + test_files[0] + '/new_data/'
test_types = [
name for name in os.listdir(test_path)
if os.path.isdir(os.path.join(test_path, name))
]
result_data = []
if (custom):
for _ in clusterings:
result_data.append([])
for test in test_types:
avg_der = 0
avg_prec = 0
avg_rec = 0
avg_cov = 0
avg_pur = 0
counter = 0
speaker_results = {}
cluster_results = []
speaker_results_cluster = []
for _ in clusterings:
cluster_results.append({
'der': 0,
'prec': 0,
'rec': 0,
'cov': 0,
'pur': 0
})
speaker_results_cluster.append({})
for f in test_files:
test_file = base_test + f
data_file = test_file + '/new_data/' + f + '.json'
with open(data_file) as f:
data = json.load(f)
for sub_f in data:
counter += 1
sub_f_data = data[sub_f]
true_labels = sub_f_data['labels']
true_speakers = sub_f_data['no_speakers']
speakers_int = OrderedSet(
map(lambda x: x['speaker'], true_labels))
for i, s in enumerate(speakers_int):
for datadict in true_labels:
if datadict['speaker'] == s:
datadict['speaker'] = i + 1
true_annotation = convert_to_annotation(true_labels)
pred_path = test_file + '/new_data/' + test + '/'
pred_file = sub_f if test == 'default' else sub_f.split(
'.')[0] + '_' + test + '.wav'
audio = {'uri': pred_file, 'audio': pred_path + pred_file}
if (custom):
long_turns, _, _, embeddings = predict(audio)
index = 0
for algorithm in clusterings:
if (custom):
pred_annotation = cluster_annotation(
long_turns, embeddings, true_speakers,
algorithm)
if (type(pred_annotation) is tuple
or pred_annotation == Annotation()):
continue
pred_annotation = pred_annotation.rename_labels(
generator='int')
der_res = der(true_annotation, pred_annotation)
prec_res = prec(true_annotation, pred_annotation)
rec_res = recall(true_annotation, pred_annotation)
cov_res = coverage(true_annotation,
pred_annotation)
pur_res = purity(true_annotation, pred_annotation)
cluster_results[index]['der'] += der_res
cluster_results[index]['prec'] += prec_res
cluster_results[index]['rec'] += rec_res
cluster_results[index]['cov'] += cov_res
cluster_results[index]['pur'] += pur_res
if not true_speakers in speaker_results_cluster[
index]:
speaker_results_cluster[index][
true_speakers] = {
'der': 0,
'prec': 0,
'rec': 0,
'cov': 0,
'pur': 0,
'counter': 0
}
speaker_results_cluster[index][true_speakers][
'der'] += der_res
speaker_results_cluster[index][true_speakers][
'prec'] += prec_res
speaker_results_cluster[index][true_speakers][
'rec'] += rec_res
speaker_results_cluster[index][true_speakers][
'cov'] += cov_res
speaker_results_cluster[index][true_speakers][
'pur'] += pur_res
speaker_results_cluster[index][true_speakers][
'counter'] += 1
index += 1
else:
pred_annotation = pipeline(
{'audio': pred_path + pred_file})
der_res = der(true_annotation, pred_annotation)
prec_res = prec(true_annotation, pred_annotation)
rec_res = recall(true_annotation, pred_annotation)
cov_res = coverage(true_annotation, pred_annotation)
pur_res = purity(true_annotation, pred_annotation)
avg_der += der_res
avg_prec += prec_res
avg_rec += rec_res
avg_cov += cov_res
avg_pur += pur_res
if not true_speakers in speaker_results:
speaker_results[true_speakers] = {
'der': 0,
'prec': 0,
'rec': 0,
'cov': 0,
'pur': 0,
'counter': 0
}
speaker_results[true_speakers]['der'] += der_res
speaker_results[true_speakers]['prec'] += prec_res
speaker_results[true_speakers]['rec'] += rec_res
speaker_results[true_speakers]['cov'] += cov_res
speaker_results[true_speakers]['pur'] += pur_res
speaker_results[true_speakers]['counter'] += 1
if custom:
index = 0
for algorithm in clusterings:
cluster_data = cluster_results[index]
sub_data = {'type': test}
sub_data['DER'] = cluster_data['der'] / counter
sub_data['Precision'] = cluster_data['prec'] / counter
sub_data['Recall'] = cluster_data['rec'] / counter
sub_data['Coverage'] = cluster_data['cov'] / counter
sub_data['Purity'] = cluster_data['pur'] / counter
for s in speaker_results_cluster[index]:
speaker_results_cluster[index][s]['der'] = speaker_results_cluster[index][s]['der'] / \
speaker_results_cluster[index][s]['counter']
speaker_results_cluster[index][s]['prec'] = speaker_results_cluster[index][s]['prec'] / \
speaker_results_cluster[index][s]['counter']
speaker_results_cluster[index][s]['rec'] = speaker_results_cluster[index][s]['rec'] / \
speaker_results_cluster[index][s]['counter']
speaker_results_cluster[index][s]['cov'] = speaker_results_cluster[index][s]['cov'] / \
speaker_results_cluster[index][s]['counter']
speaker_results_cluster[index][s]['pur'] = speaker_results_cluster[index][s]['pur'] / \
speaker_results_cluster[index][s]['counter']
sub_data['Speaker_data'] = speaker_results_cluster[index]
result_data[index].append(sub_data)
result[prefix + 'custom' + algorithm] = result_data[index]
index += 1
else:
sub_data = {'type': test}
sub_data['DER'] = avg_der / counter
sub_data['Precision'] = avg_prec / counter
sub_data['Recall'] = avg_rec / counter
sub_data['Coverage'] = avg_cov / counter
sub_data['Purity'] = avg_pur / counter
for s in speaker_results:
speaker_results[s]['der'] = speaker_results[s]['der'] / \
speaker_results[s]['counter']
speaker_results[s]['prec'] = speaker_results[s]['prec'] / \
speaker_results[s]['counter']
speaker_results[s]['rec'] = speaker_results[s]['rec'] / \
speaker_results[s]['counter']
speaker_results[s]['cov'] = speaker_results[s]['cov'] / \
speaker_results[s]['counter']
speaker_results[s]['pur'] = speaker_results[s]['pur'] / \
speaker_results[s]['counter']
sub_data['Speaker_data'] = speaker_results
result_data.append(sub_data)
result[prefix + 'auto'] = result_data
save_file = 'results.json'
with open(save_file, 'w') as outfile:
json.dump(result, outfile)
return result_data
def test(protocol, tune_dir, apply_dir, subset='test', beta=1.0):
os.makedirs(apply_dir)
train_dir = os.path.dirname(os.path.dirname(os.path.dirname(tune_dir)))
duration = float(os.path.basename(train_dir))
config_dir = os.path.dirname(os.path.dirname(os.path.dirname(train_dir)))
config_yml = config_dir + '/config.yml'
with open(config_yml, 'r') as fp:
config = yaml.load(fp)
# -- FEATURE EXTRACTION --
feature_extraction_name = config['feature_extraction']['name']
features = __import__('pyannote.audio.features',
fromlist=[feature_extraction_name])
FeatureExtraction = getattr(features, feature_extraction_name)
feature_extraction = FeatureExtraction(
**config['feature_extraction'].get('params', {}))
# -- HYPER-PARAMETERS --
tune_yml = tune_dir + '/tune.yml'
with open(tune_yml, 'r') as fp:
tune = yaml.load(fp)
architecture_yml = train_dir + '/architecture.yml'
WEIGHTS_H5 = train_dir + '/weights/{epoch:04d}.h5'
weights_h5 = WEIGHTS_H5.format(epoch=tune['epoch'])
sequence_embedding = SequenceEmbedding.from_disk(
architecture_yml, weights_h5)
segmentation = Segmentation(
sequence_embedding, feature_extraction,
duration=duration, step=0.100)
peak = Peak(alpha=tune['alpha'])
HARD_JSON = apply_dir + '/{uri}.hard.json'
SOFT_PKL = apply_dir + '/{uri}.soft.pkl'
eval_txt = apply_dir + '/eval.txt'
TEMPLATE = '{uri} {purity:.5f} {coverage:.5f} {f_measure:.5f}\n'
purity = SegmentationPurity()
coverage = SegmentationCoverage()
fscore = []
for test_file in getattr(protocol, subset)():
soft = segmentation.apply(test_file)
hard = peak.apply(soft)
uri = get_unique_identifier(test_file)
path = SOFT_PKL.format(uri=uri)
mkdir_p(os.path.dirname(path))
with open(path, 'w') as fp:
pickle.dump(soft, fp)
path = HARD_JSON.format(uri=uri)
mkdir_p(os.path.dirname(path))
with open(path, 'w') as fp:
pyannote.core.json.dump(hard, fp)
try:
reference = test_file['annotation']
uem = test_file['annotated']
except KeyError as e:
continue
p = purity(reference, hard)
c = coverage(reference, hard)
f = f_measure(c, p, beta=beta)
fscore.append(f)
line = TEMPLATE.format(
uri=uri, purity=p, coverage=c, f_measure=f)
with open(eval_txt, 'a') as fp:
fp.write(line)
p = abs(purity)
c = abs(coverage)
f = np.mean(fscore)
line = TEMPLATE.format(
uri='ALL', purity=p, coverage=c, f_measure=f)
with open(eval_txt, 'a') as fp:
fp.write(line)
groundtruth = {}
for test_file in protocol.development():
uri = test_file['uri']
groundtruth[uri] = test_file['annotation']
wav = test_file['medium']['wav']
# this is where the magic happens
predictions[uri] = segmentation.apply(wav)
# tested thresholds
alphas = np.linspace(0, 1, 50)
# evaluation metrics (purity and coverage)
from pyannote.metrics.segmentation import SegmentationPurity
from pyannote.metrics.segmentation import SegmentationCoverage
purity = [SegmentationPurity() for alpha in alphas]
coverage = [SegmentationCoverage() for alpha in alphas]
# peak detection
from pyannote.audio.signal import Peak
for i, alpha in enumerate(alphas):
# initialize peak detection algorithm
peak = Peak(alpha=alpha, min_duration=1.0)
for uri, reference in groundtruth.items():
# apply peak detection
hypothesis = peak.apply(predictions[uri])
# compute purity and coverage
purity[i](reference, hypothesis)
coverage[i](reference, hypothesis)
# print the results in three columns:
# threshold, purity, coverage
def evaluate(protocol,
train_dir,
store_dir,
subset='development',
epoch=None,
min_duration=1.0):
mkdir_p(store_dir)
# -- LOAD MODEL --
nb_epoch = 0
while True:
weights_h5 = LoggingCallback.WEIGHTS_H5.format(log_dir=train_dir,
epoch=nb_epoch)
if not os.path.isfile(weights_h5):
break
nb_epoch += 1
config_dir = os.path.dirname(os.path.dirname(train_dir))
config_yml = config_dir + '/config.yml'
with open(config_yml, 'r') as fp:
config = yaml.load(fp)
# -- FEATURE EXTRACTION --
feature_extraction_name = config['feature_extraction']['name']
features = __import__('pyannote.audio.features',
fromlist=[feature_extraction_name])
FeatureExtraction = getattr(features, feature_extraction_name)
feature_extraction = FeatureExtraction(
**config['feature_extraction'].get('params', {}))
# -- SEQUENCE GENERATOR --
duration = config['sequences']['duration']
step = config['sequences']['step']
groundtruth = {}
for dev_file in getattr(protocol, subset)():
uri = dev_file['uri']
groundtruth[uri] = dev_file['annotation']
# -- CHOOSE MODEL --
if epoch > nb_epoch:
raise ValueError('Epoch should be less than ' + str(nb_epoch))
if epoch is None:
epoch = nb_epoch - 1
sequence_labeling = SequenceLabeling.from_disk(train_dir, epoch)
aggregation = SequenceLabelingAggregation(sequence_labeling,
feature_extraction,
duration=duration,
step=step)
# -- PREDICTION --
predictions = {}
for dev_file in getattr(protocol, subset)():
uri = dev_file['uri']
predictions[uri] = aggregation.apply(dev_file)
alphas = np.linspace(0, 1, 20)
purity = [SegmentationPurity(parallel=False) for alpha in alphas]
coverage = [SegmentationCoverage(parallel=False) for alpha in alphas]
# -- SAVE RESULTS --
for i, alpha in enumerate(alphas):
# initialize peak detection algorithm
peak = Peak(alpha=alpha, min_duration=min_duration)
for uri, reference in groundtruth.items():
# apply peak detection
hypothesis = peak.apply(predictions[uri])
# compute purity and coverage
purity[i](reference, hypothesis)
coverage[i](reference, hypothesis)
TEMPLATE = '{alpha:g} {purity:.3f}% {coverage:.3f}%'
with open(store_dir + '/res.txt', 'a') as fp:
for i, a in enumerate(alphas):
p = 100 * abs(purity[i])
c = 100 * abs(coverage[i])
print(TEMPLATE.format(alpha=a, purity=p, coverage=c))
fp.write(TEMPLATE.format(alpha=a, purity=p, coverage=c) + '\n')