def objective_function(parameters, beta=1.0):
epoch = parameters[0]
weights_h5 = WEIGHTS_H5.format(epoch=epoch)
sequence_embedding = SequenceEmbedding.from_disk(
architecture_yml, weights_h5)
fX = sequence_embedding.transform(X, batch_size=batch_size)
# compute distance between every pair of sequences
y_distance = pdist(fX, metric=distance)
# compute same/different groundtruth
y_true = pdist(y, metric='chebyshev') < 1
# false positive / true positive
fpr, tpr, thresholds = sklearn.metrics.roc_curve(
y_true, -y_distance, pos_label=True, drop_intermediate=True)
fnr = 1. - tpr
far = fpr
thresholds = -thresholds
fscore = 1. - f_measure(1. - fnr, 1. - far, beta=beta)
i = np.nanargmin(fscore)
alphas[epoch] = float(thresholds[i])
return fscore[i]
def objective_function(parameters, beta=1.0):
epoch, alpha = parameters
weights_h5 = WEIGHTS_H5.format(epoch=epoch)
sequence_embedding = SequenceEmbedding.from_disk(
architecture_yml, weights_h5)
segmentation = Segmentation(
sequence_embedding, feature_extraction,
duration=duration, step=0.100)
if epoch not in predictions:
predictions[epoch] = {}
purity = SegmentationPurity()
coverage = SegmentationCoverage()
f, n = 0., 0
for dev_file in getattr(protocol, subset)():
uri = get_unique_identifier(dev_file)
reference = dev_file['annotation']
n += 1
if uri in predictions[epoch]:
prediction = predictions[epoch][uri]
else:
prediction = segmentation.apply(dev_file)
predictions[epoch][uri] = prediction
peak = Peak(alpha=alpha)
hypothesis = peak.apply(prediction)
p = purity(reference, hypothesis)
c = coverage(reference, hypothesis)
f += f_measure(c, p, beta=beta)
return 1 - (f / n)
def test(protocol, tune_dir, test_dir, subset, beta=1.0):
batch_size = 32
try:
os.makedirs(test_dir)
except Exception as e:
pass
train_dir = os.path.dirname(os.path.dirname(tune_dir))
# -- DURATIONS --
duration, min_duration, step, heterogeneous = \
path_to_duration(os.path.basename(train_dir))
config_dir = os.path.dirname(os.path.dirname(os.path.dirname(train_dir)))
config_yml = config_dir + '/config.yml'
with open(config_yml, 'r') as fp:
config = yaml.load(fp)
# -- PREPROCESSORS --
for key, preprocessor in config.get('preprocessors', {}).items():
preprocessor_name = preprocessor['name']
preprocessor_params = preprocessor.get('params', {})
preprocessors = __import__('pyannote.audio.preprocessors',
fromlist=[preprocessor_name])
Preprocessor = getattr(preprocessors, preprocessor_name)
protocol.preprocessors[key] = Preprocessor(**preprocessor_params)
# -- FEATURE EXTRACTION --
feature_extraction_name = config['feature_extraction']['name']
features = __import__('pyannote.audio.features',
fromlist=[feature_extraction_name])
FeatureExtraction = getattr(features, feature_extraction_name)
feature_extraction = FeatureExtraction(
**config['feature_extraction'].get('params', {}))
distance = config['glue'].get('params', {}).get('distance', 'sqeuclidean')
# -- HYPER-PARAMETERS --
tune_yml = tune_dir + '/tune.yml'
with open(tune_yml, 'r') as fp:
tune = yaml.load(fp)
architecture_yml = train_dir + '/architecture.yml'
WEIGHTS_H5 = train_dir + '/weights/{epoch:04d}.h5'
weights_h5 = WEIGHTS_H5.format(epoch=tune['epoch'])
sequence_embedding = SequenceEmbedding.from_disk(
architecture_yml, weights_h5)
X, y = generate_test(protocol, subset, feature_extraction,
duration, min_duration=min_duration, step=step)
fX = sequence_embedding.transform(X, batch_size=batch_size)
y_distance = pdist(fX, metric=distance)
y_true = pdist(y, metric='chebyshev') < 1
fpr, tpr, thresholds = sklearn.metrics.roc_curve(
y_true, -y_distance, pos_label=True, drop_intermediate=True)
frr = 1. - tpr
far = fpr
thresholds = -thresholds
eer_index = np.where(far > frr)[0][0]
eer = .25 * (far[eer_index-1] + far[eer_index] +
frr[eer_index-1] + frr[eer_index])
fscore = 1. - f_measure(1. - frr, 1. - far, beta=beta)
opt_i = np.nanargmin(fscore)
opt_alpha = float(thresholds[opt_i])
opt_far = far[opt_i]
opt_frr = frr[opt_i]
opt_fscore = fscore[opt_i]
alpha = tune['alpha']
actual_i = np.searchsorted(thresholds, alpha)
actual_far = far[actual_i]
actual_frr = frr[actual_i]
actual_fscore = fscore[actual_i]
save_to = test_dir + '/' + subset
plot_distributions(y_true, y_distance, save_to)
eer = plot_det_curve(y_true, -y_distance, save_to)
plot_precision_recall_curve(y_true, -y_distance, save_to)
with open(save_to + '.txt', 'w') as fp:
fp.write('# cond. thresh far frr fscore eer\n')
TEMPLATE = '{condition} {alpha:.5f} {far:.5f} {frr:.5f} {fscore:.5f} {eer:.5f}\n'
fp.write(TEMPLATE.format(condition='optimal',
alpha=opt_alpha,
far=opt_far,
frr=opt_frr,
fscore=opt_fscore,
eer=eer))
fp.write(TEMPLATE.format(condition='actual ',
alpha=alpha,
far=actual_far,
frr=actual_frr,
fscore=actual_fscore,
eer=eer))
def test(dataset, medium_template, config_yml, weights_h5, output_dir):
# load configuration file
with open(config_yml, 'r') as fp:
config = yaml.load(fp)
# this is where model architecture was saved
architecture_yml = os.path.dirname(
os.path.dirname(weights_h5)) + '/architecture.yml'
# -- DATASET --
db, task, protocol, subset = dataset.split('.')
database = get_database(db, medium_template=medium_template)
protocol = database.get_protocol(task, protocol)
if not hasattr(protocol, subset):
raise NotImplementedError('')
file_generator = getattr(protocol, subset)()
# -- FEATURE EXTRACTION --
# input sequence duration
duration = config['feature_extraction']['duration']
# MFCCs
feature_extractor = YaafeMFCC(**config['feature_extraction']['mfcc'])
# normalization
normalize = config['feature_extraction']['normalize']
# -- TESTING --
# overlap ratio between each window
overlap = config['testing']['overlap']
step = duration * (1. - overlap)
# prediction smoothing
onset = config['testing']['binarize']['onset']
offset = config['testing']['binarize']['offset']
binarizer = Binarize(onset=0.5, offset=0.5)
sequence_labeling = SequenceLabeling.from_disk(architecture_yml,
weights_h5)
aggregation = SequenceLabelingAggregation(sequence_labeling,
feature_extractor,
normalize=normalize,
duration=duration,
step=step)
collar = 0.500
error_rate = DetectionErrorRate(collar=collar)
accuracy = DetectionAccuracy(collar=collar)
precision = DetectionPrecision(collar=collar)
recall = DetectionRecall(collar=collar)
LINE = '{uri} {e:.3f} {a:.3f} {p:.3f} {r:.3f} {f:.3f}\n'
PATH = '{output_dir}/eval.{dataset}.{subset}.txt'
path = PATH.format(output_dir=output_dir, dataset=dataset, subset=subset)
with open(path, 'w') as fp:
header = '# uri error accuracy precision recall f_measure\n'
fp.write(header)
fp.flush()
for current_file in file_generator:
uri = current_file['uri']
wav = current_file['medium']['wav']
annotated = current_file['annotated']
annotation = current_file['annotation']
predictions = aggregation.apply(wav)
hypothesis = binarizer.apply(predictions, dimension=1)
e = error_rate(annotation, hypothesis, uem=annotated)
a = accuracy(annotation, hypothesis, uem=annotated)
p = precision(annotation, hypothesis, uem=annotated)
r = recall(annotation, hypothesis, uem=annotated)
f = f_measure(p, r)
line = LINE.format(uri=uri, e=e, a=a, p=p, r=r, f=f)
fp.write(line)
fp.flush()
PATH = '{output_dir}/{uri}.json'
path = PATH.format(output_dir=output_dir, uri=uri)
dump_to(hypothesis, path)
# average on whole corpus
uri = '{dataset}.{subset}'.format(dataset=dataset, subset=subset)
e = abs(error_rate)
a = abs(accuracy)
p = abs(precision)
r = abs(recall)
f = f_measure(p, r)
line = LINE.format(uri=uri, e=e, a=a, p=p, r=r, f=f)
fp.write(line)
fp.flush()
def test(protocol, tune_dir, apply_dir, subset='test', beta=1.0):
os.makedirs(apply_dir)
train_dir = os.path.dirname(os.path.dirname(os.path.dirname(tune_dir)))
duration = float(os.path.basename(train_dir))
config_dir = os.path.dirname(os.path.dirname(os.path.dirname(train_dir)))
config_yml = config_dir + '/config.yml'
with open(config_yml, 'r') as fp:
config = yaml.load(fp)
# -- FEATURE EXTRACTION --
feature_extraction_name = config['feature_extraction']['name']
features = __import__('pyannote.audio.features',
fromlist=[feature_extraction_name])
FeatureExtraction = getattr(features, feature_extraction_name)
feature_extraction = FeatureExtraction(
**config['feature_extraction'].get('params', {}))
# -- HYPER-PARAMETERS --
tune_yml = tune_dir + '/tune.yml'
with open(tune_yml, 'r') as fp:
tune = yaml.load(fp)
architecture_yml = train_dir + '/architecture.yml'
WEIGHTS_H5 = train_dir + '/weights/{epoch:04d}.h5'
weights_h5 = WEIGHTS_H5.format(epoch=tune['epoch'])
sequence_embedding = SequenceEmbedding.from_disk(
architecture_yml, weights_h5)
segmentation = Segmentation(
sequence_embedding, feature_extraction,
duration=duration, step=0.100)
peak = Peak(alpha=tune['alpha'])
HARD_JSON = apply_dir + '/{uri}.hard.json'
SOFT_PKL = apply_dir + '/{uri}.soft.pkl'
eval_txt = apply_dir + '/eval.txt'
TEMPLATE = '{uri} {purity:.5f} {coverage:.5f} {f_measure:.5f}\n'
purity = SegmentationPurity()
coverage = SegmentationCoverage()
fscore = []
for test_file in getattr(protocol, subset)():
soft = segmentation.apply(test_file)
hard = peak.apply(soft)
uri = get_unique_identifier(test_file)
path = SOFT_PKL.format(uri=uri)
mkdir_p(os.path.dirname(path))
with open(path, 'w') as fp:
pickle.dump(soft, fp)
path = HARD_JSON.format(uri=uri)
mkdir_p(os.path.dirname(path))
with open(path, 'w') as fp:
pyannote.core.json.dump(hard, fp)
try:
reference = test_file['annotation']
uem = test_file['annotated']
except KeyError as e:
continue
p = purity(reference, hard)
c = coverage(reference, hard)
f = f_measure(c, p, beta=beta)
fscore.append(f)
line = TEMPLATE.format(
uri=uri, purity=p, coverage=c, f_measure=f)
with open(eval_txt, 'a') as fp:
fp.write(line)
p = abs(purity)
c = abs(coverage)
f = np.mean(fscore)
line = TEMPLATE.format(
uri='ALL', purity=p, coverage=c, f_measure=f)
with open(eval_txt, 'a') as fp:
fp.write(line)