from spdr.metrics import SPDR_Metrics
from spdr.parser.voxceleb_parser import Voxceleb_Parser
vp = Voxceleb_Parser('./data/groundtruth/voxceleb1')
vp.parse()
ref = vp.get_Reference()
spdr_metrics = SPDR_Metrics(
ref['_dyeYOti1_E'], ref['_dyeYOti1_E']) #second arg should be hypothesis
print(spdr_metrics.get_DiarizationErrorRate(detailed=True))
spdr_metrics.get_Plot()
from spdr.metrics import SPDR_Metrics
from pyannote.core import Segment, Timeline, Annotation
# get metrics
reference = Annotation(uri='file2')
reference[Segment(0, 5)] = 'A'
reference[Segment(6, 10)] = 'B'
reference[Segment(12, 13)] = 'B'
reference[Segment(15, 20)] = 'A'
hypothesis = Annotation(uri='file2')
hypothesis[Segment(0, 5)] = 'a'
hypothesis[Segment(6, 10)] = 'b'
hypothesis[Segment(12, 13)] = 'b'
hypothesis[Segment(15, 20)] = 'a'
uem = Timeline([Segment(float(6), float(12))])
spdr_metrics = SPDR_Metrics(reference, hypothesis, uem)
print("DER simple")
print(spdr_metrics.get_DiarizationErrorRate(detailed=False))
print("DER detailed")
print(spdr_metrics.get_DiarizationErrorRate(detailed=True))
print("DER simple greedy")
print(spdr_metrics.get_DiarizationErrorRate(greedy=True, detailed=False))
print("DER detailed greedy")
print(spdr_metrics.get_DiarizationErrorRate(greedy=True, detailed=True))
spdr_metrics.get_Plot()
print("")
import pickle
from collections import Counter
from spdr.parser.hypothesis_generator import define_hypothesis
from spdr.metrics import SPDR_Metrics
from pyannote.core import Timeline
with open('dump.pkl', 'rb') as pFile:
data = pickle.load(pFile)
sequences = data['sequences']
cluster_results = data['cluster_results']
uem = data['uem']
print(uem.for_json())
uem = Timeline().from_json({
'pyannote': 'Timeline',
'content': [{
'end': 1160.0,
'start': 1144.0
}]
})
#print(Counter(cluster_results))
h = define_hypothesis(sequences, cluster_results, uem)
spdr_metrics = SPDR_Metrics(h, h, uem)
spdr_metrics.get_Plot(80)
def run(self, ret=False):
# check audio specification if needed
handler = SPDR_RT09_Handler()
dataset = handler.run()
dataset = dataset[0] if type(
dataset
) is list else dataset # this should be removed once a voting system is implemented
condition = self.config['spkr']['condition']
mdm_warning = """MDM relies on some sort of voting system for all the microphone recordings to perform well.\n
This feature is not yet implemented and the MDM condition will currently yield bad results.\n
You might want to use SDM condition instead!"""
print(mdm_warning if condition.lower() == 'mdm' else '')
print("SPEAKER COUNT: %d" % len(dataset["speakers"]))
file_to_progress = (dataset["files"][0])
if self.config["data"]["use_normalized_audio"]:
file_to_progress = file_to_progress.replace(
".wav", "_normalized.wav")
if not os.path.isfile(file_to_progress):
raise Exception(
"Normalized audio file %s not found. Please run the normalize_audio.sh first."
)
aligned_timeline_in_ms = Controller._align_timeline(
self.config['segment']['size'], (dataset["start"], dataset["end"]))
embedding_pickle = os.path.join(
self.config["data"]["pickle"],
get_filename_with_new_extension(file_to_progress, ".pkl"))
if os.path.isfile(embedding_pickle):
with open(embedding_pickle, 'rb') as pFile:
embeddings = pickle.load(pFile)
else:
# split in sequences
segment_wave_files_from_to(
files=[file_to_progress],
segment_path=self.config['data']['out'],
segment_size=self.config['segment']['size'],
timeline=aligned_timeline_in_ms)
# get embedding vector - use https://github.com/stdm/ZHAW_deep_voice/
embeddings = self.embedding_extractor.extract_embeddings(
[file_to_progress])
with open(embedding_pickle, 'wb') as pFile:
pickle.dump(embeddings, pFile)
# join similar sequences (i.e. part A of sentence 1 with part B of sentence 1)
sequencer = SPDR_Sequencer(embeddings=embeddings)
changepoint_vector = sequencer.find_changepoints()
print("Number of changepoints: %d" % changepoint_vector.count(0))
# pass into clustering
clustering = clustering_controller.ClusteringController(self.config)
cluster_result = clustering.cluster_embeddings(
embeddings,
np.array(range(0, len(embeddings))),
changepoint_vector=changepoint_vector)
print("Number of clusters: %d" % (max(cluster_result) + 1))
# run VAD
segment_folder = os.path.join(
self.config['data']['out'],
get_filename_without_extension(os.path.basename(file_to_progress)))
self.vad.classify_clusters(segment_folder, cluster_result,
self.config['vad']['aggressiveness'])
# get metrics
uem = Timeline([Segment(dataset["start"], dataset["end"])])
hypothesis = define_hypothesis_for_embeddings(
cluster_result, aligned_timeline_in_ms[0] / 1000)
# do cleanup
handler.do_cleanup()
if not ret:
spdr_metrics = SPDR_Metrics(dataset["reference"], hypothesis, uem)
metrics = spdr_metrics.get_DiarizationErrorRate(detailed=True)
print('=' * 15, 'Results', '=' * 15)
print('DER:\t\t\t%.4f' % metrics['diarization error rate'])
print('Confusion:\t\t%.4f' % metrics['confusion'])
print('False Alarm:\t\t%.4f' % metrics['false alarm'])
print('Correct:\t\t%.4f' % metrics['correct'])
print('Missed Detection:\t%.4f' % metrics['missed detection'])
print('Total:\t\t\t%.4f' % metrics['total'])
print('=' * 39)
der = spdr_metrics.get_DiarizationErrorRate(detailed=False)
purity = spdr_metrics.get_DiarizationPurity(detailed=False)
coverage = spdr_metrics.get_DiarizationCoverage(detailed=False)
cpfscore = spdr_metrics.get_DiarizationCoveragePurityFScore(
detailed=False)
confusion = metrics['confusion']
falsealarm = metrics['false alarm']
correct = metrics['correct']
misseddetection = metrics['missed detection']
ptext = "DER: %.4f | Purity: %.4f | Coverage: %.4f | PC-F-Score: %.4f \nConfusion: %.2f | False Alarm: %.2f | Correct: %.2f | Missed Detection %.2f" % (\
der, \
purity, \
coverage, \
cpfscore, \
confusion, \
falsealarm, \
correct, \
misseddetection
)
spdr_metrics.get_Plot(title=dataset['id'], text=ptext)
else:
return {
'dataset': dataset,
'reference': dataset["reference"],
'hypothesis': hypothesis,
'uem': uem,
'cluster_result': cluster_result,
'aligned_timeline_in_ms': aligned_timeline_in_ms[0] / 1000
}
hypothesis2[Segment(6, 7)] = 'b'
hypothesis2[Segment(7, 8)] = 'c'
hypothesis2[Segment(8, 10)] = 'd'
hypothesis2[Segment(12, 13)] = 'b'
hypothesis2[Segment(15, 17)] = 'c'
hypothesis2[Segment(17, 18)] = 'e'
hypothesis2[Segment(18, 20)] = 'c'
hypothesis3 = Annotation()
hypothesis3[Segment(0, 5)] = 'a'
hypothesis3[Segment(6, 10)] = 'a'
hypothesis3[Segment(12, 13)] = 'a'
hypothesis3[Segment(15, 20)] = 'a'
uem = Timeline([Segment(float(0), float(20))])
spdr_metrics1 = SPDR_Metrics(reference, hypothesis1, uem)
spdr_metrics2 = SPDR_Metrics(reference, hypothesis2, uem)
spdr_metrics3 = SPDR_Metrics(reference, hypothesis3, uem)
text1 = "\nDER: %.4f | Purity: %.4f | Coverage: %.4f | PC-F-Score: %.4f" % (
spdr_metrics1.get_DiarizationErrorRate(detailed=False),
spdr_metrics1.get_DiarizationPurity(detailed=False),
spdr_metrics1.get_DiarizationCoverage(detailed=False),
spdr_metrics1.get_DiarizationCoveragePurityFScore(detailed=False))
text2 = "\nDER: %.4f | Purity: %.4f | Coverage: %.4f | PC-F-Score: %.4f" % (
spdr_metrics2.get_DiarizationErrorRate(detailed=False),
spdr_metrics2.get_DiarizationPurity(detailed=False),
spdr_metrics2.get_DiarizationCoverage(detailed=False),
spdr_metrics2.get_DiarizationCoveragePurityFScore(detailed=False))
text3 = "\nDER: %.4f | Purity: %.4f | Coverage: %.4f | PC-F-Score: %.4f" % (
spdr_metrics3.get_DiarizationErrorRate(detailed=False),
hypothesis1[Segment(13, 20)] = 'c'
purity = Annotation()
purity[Segment(0, 6)] = 'A'
purity[Segment(10, 12)] = 'A'
purity[Segment(13, 20)] = 'A'
coverage = Annotation()
coverage[Segment(0, 6)] = 'B'
coverage[Segment(6, 10)] = 'B'
coverage[Segment(13, 20)] = 'B'
precisionrecall = Annotation()
uem = Timeline([Segment(float(0), float(20))])
spdr_metrics = SPDR_Metrics(reference, hypothesis1, uem)
text1 = "\nDER: %.4f | Purity: %.4f | Coverage: %.4f | PC-F-Score: %.4f" % (\
spdr_metrics.get_DiarizationErrorRate(detailed=False), \
spdr_metrics.get_DiarizationPurity(detailed=False), \
spdr_metrics.get_DiarizationCoverage(detailed=False), \
spdr_metrics.get_DiarizationCoveragePurityFScore(detailed=False), \
)
# plot reference
plt.rcParams['figure.figsize'] = (10, 9)
plt.subplot(311)
notebook.plot_annotation(reference, legend=True, time=True)
plt.gca().text(0.6,
0.15,
'Reference',
Segment(
float(dataset["start"]) / 1000,
float(dataset["end"]) / 1000)
])
else:
uem = Timeline(
[Segment(float(dataset["start"]), float(dataset["end"]))])
non_speech = np.random.choice(list(set(cluster_result)))
#non_speech = list(set(np.random.choice(cluster_result, np.random.randint(0,len(cluster_result)))))
# note: for a re run, this might need to be reworked, because the current API does not allow to specify a non_speech list
hypothesis = define_hypothesis_for_embeddings(cluster_result,
non_speech)
spdr_metrics = SPDR_Metrics(dataset["reference"], hypothesis, uem)
metrics = spdr_metrics.get_DiarizationErrorRate(detailed=True)
print('=' * 15, 'Results', '=' * 15)
print('=' * 10, dataset['id'], '=' * 10)
print('DER:\t\t\t%.4f' % metrics['diarization error rate'])
print('Confusion:\t\t%.4f' % metrics['confusion'])
print('False Alarm:\t\t%.4f' % metrics['false alarm'])
print('Correct:\t\t%.4f' % metrics['correct'])
print('Missed Detection:\t%.4f' % metrics['missed detection'])
print('Total:\t\t\t%.4f' % metrics['total'])
print('Purity:\t\t\t%.4f' % spdr_metrics.get_DiarizationPurity())
print('Coverage:\t\t%.4f' % spdr_metrics.get_DiarizationCoverage())
print('=' * 39)
results.append([
metrics['diarization error rate'], metrics['confusion'],
metrics['false alarm'], metrics['correct'],
network = experiment_['embeddings']['network']
ds = ds.replace(IN_PATH, '').replace('/', '')
network = network.replace('.h5', '')
experiment_name = "DATASET_%s.SUBSET_%s.MODEL_%s" % (
ds.lower(), ds_subset.lower(), network.lower())
run_folder = os.path.join(RUN_PATH, experiment_name)
print('=' * 20, 'Plotting experiment', idx + 1, 'of', len(experiments_),
'=' * 20)
print('-' * 10, 'Name:', experiment_name, '-' * 10)
# Load
RESULTS_PICKLE = os.path.join(run_folder, 'result.pkl')
with open(RESULTS_PICKLE, 'rb') as pFile:
results = pickle.load(pFile)
try:
spdr_metrics = SPDR_Metrics(results["reference"],
results["hypothesis"], results["uem"])
dataset = results['dataset']
metrics = spdr_metrics.get_DiarizationErrorRate(detailed=True)
der = spdr_metrics.get_DiarizationErrorRate(detailed=False)
purity = spdr_metrics.get_DiarizationPurity(detailed=False)
coverage = spdr_metrics.get_DiarizationCoverage(detailed=False)
cpfscore = spdr_metrics.get_DiarizationCoveragePurityFScore(
detailed=False)
confusion = metrics['confusion']
falsealarm = metrics['false alarm']
correct = metrics['correct']
misseddetection = metrics['missed detection']
total = metrics['total']
ptext = "DER: %.4f | Purity: %.4f | Coverage: %.4f | PC-F-Score: %.4f \nConfusion: %.2f | False Alarm: %.2f | Correct: %.2f | Missed Detection %.2f" % (\
der, \