def evaluate_per_file(expid, file_list, audio_hop=882, sr=44100,
sequence_frames=128,
sequence_hop=64, normalize_data=True, mel_bands=40,
load_predictions=True):
modelfolder = (
'/scratch/js7561/datasets/scaper_waspaa2017/urban-sed/models')
expfolder = os.path.join(modelfolder, expid)
if load_predictions:
ytestfile = os.path.join(expfolder, 'ytest.npy.gz')
yprobfile = os.path.join(expfolder, 'yprob.npy.gz')
yidfile = os.path.join(expfolder, 'yid.npy.gz')
y_test = np.load(gzip.open(ytestfile, 'rb'))
pred_test = np.load(gzip.open(yprobfile, 'rb'))
id_test = np.load(gzip.open(yidfile, 'rb'))
label_list = (['air_conditioner', 'car_horn', 'children_playing',
'dog_bark', 'drilling', 'engine_idling', 'gun_shot',
'jackhammer', 'siren', 'street_music'])
label_list = sorted(label_list)
else:
# Build model and load in best weights
model = build_crnn_onestep(n_freq_cnn=mel_bands)
model.compile(optimizer='adam', loss='binary_crossentropy',
metrics=['accuracy'])
weightfile = os.path.join(expfolder, 'weights_best.hdf5'.format(expid))
model.load_weights(weightfile)
# Load data
(x_train, y_train, id_train, x_val, y_val, id_val, x_test, y_test,
id_test, label_list, scaler) = load_urbansed_crnn(
sequence_frames=sequence_frames, sequence_hop=sequence_hop,
sequence_offset=0,
audio_hop=audio_hop, sr=sr,
verbose=False, normalize=normalize_data,
mel_bands=mel_bands)
pred_test = model.predict(x_test)
# ONLY KEEP SAMPLES THAT ARE INCLUDED IN THE FILE LIST!
y_test_filtered = []
pred_test_filtered = []
id_test_filtered = []
for p, y, i in zip(pred_test, y_test, id_test):
filename = os.path.basename(i[0]).replace('.npy.gz', '')
if filename in file_list:
y_test_filtered.append(y)
pred_test_filtered.append(p)
id_test_filtered.append(i)
y_test_filtered = np.asarray(y_test_filtered)
pred_test_filtered = np.asarray(pred_test_filtered)
id_test_filtered = np.asarray(id_test_filtered)
SegMetrics1s = sed_eval.sound_event.SegmentBasedMetrics(
label_list, time_resolution=1.0)
SegMetrics100ms = sed_eval.sound_event.SegmentBasedMetrics(
label_list, time_resolution=0.1)
# DO EVAL FILE BY FILES
idx = 0
while idx < len(id_test_filtered):
y_test_file = []
pred_test_file = []
id_test_file = []
current_id = id_test_filtered[idx][0]
while idx < len(id_test_filtered) and id_test_filtered[idx][
0] == current_id:
y_test_file.append(y_test_filtered[idx])
pred_test_file.append(pred_test_filtered[idx])
id_test_file.append(id_test_filtered[idx])
idx += 1
y_test_file = np.asarray(y_test_file)
pred_test_file = np.asarray(pred_test_file)
id_test_file = np.asarray(id_test_file)
est_roll, ref_roll = combine_event_rolls(pred_test_file,
y_test_file,
threshold=0.5)
# COMPUTE FOLD RESULTS AND REPORT
# Convert event rolls into even lists
est_event_list = event_roll_to_event_list(
est_roll, label_list, audio_hop/float(sr))
ref_event_list = event_roll_to_event_list(
ref_roll, label_list, audio_hop/float(sr))
# Compute metrics at 1s and 100ms levels
SegMetrics1s.evaluate(ref_event_list, est_event_list)
SegMetrics100ms.evaluate(ref_event_list, est_event_list)
results1s = SegMetrics1s.results()
results100ms = SegMetrics100ms.results()
# Report scores
scores = (
'F1s: {:.4f}, P1s: {:.4f}, R1s: {:.4f}, F100ms: {:.4f}, '
'P100ms: {:.4f}, R100ms: {:.4f} | E1s: {:.4f}, '
'E100ms: {:.4f}'.format(
results1s['overall']['f_measure']['f_measure'],
results1s['overall']['f_measure']['precision'],
results1s['overall']['f_measure']['recall'],
results100ms['overall']['f_measure']['f_measure'],
results100ms['overall']['f_measure']['precision'],
results100ms['overall']['f_measure']['recall'],
results1s['overall']['error_rate']['error_rate'],
results100ms['overall']['error_rate']['error_rate']
))
print(scores)
# Save scores
results_all = {'results1s': results1s, 'results100ms': results100ms}
results_all_file = os.path.join(expfolder, 'test_results_per_file.json')
json.dump(results_all, open(results_all_file, 'w'), indent=2)
return results_all
def run_experiment_sedeval(expid,
epochs=1000,
metrics=['accuracy'],
sequence_hop=64,
batch_size=64,
audio_hop=882,
sr=44100,
sequence_frames=128,
epoch_limit=2048,
sed_early_stopping=100,
normalize_data=True,
fit_verbose=True,
mel_bands=40,
resume=False,
resume_f1_best=0,
load_subset=None):
# Print out library versions
print('Module versions:')
print('keras version: {:s}'.format(keras.__version__))
print('tensorflow version: {:s}'.format(tensorflow.__version__))
print('numpy version: {:s}'.format(np.__version__))
print('sklearn version: {:s}'.format(sklearn.__version__))
# Create output folders
modelfolder = '/scratch/js7561/datasets/scaper_waspaa2017/urban-sed/models'
expfolder = os.path.join(modelfolder, expid)
time.sleep(np.random.rand() * 10) # prevents jobs clashing
if not os.path.isdir(expfolder):
os.mkdir(expfolder)
# Build model
print('\nBuilding model...')
# model = build_crnn(n_freq_cnn=mel_bands)
model = build_crnn_onestep(n_freq_cnn=mel_bands)
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=metrics)
if resume:
print('Loading best weights and resuming...')
weights_best_file = os.path.join(expfolder, 'weights_best.hdf5')
model.load_weights(weights_best_file)
# Load data
(x_train, y_train, id_train, x_val, y_val, id_val, x_test, y_test, id_test,
label_list, scaler) = load_urbansed_crnn(sequence_frames=sequence_frames,
sequence_hop=sequence_hop,
sequence_offset=0,
audio_hop=audio_hop,
sr=sr,
verbose=True,
normalize=normalize_data,
mel_bands=mel_bands,
load_subset=load_subset)
# Save scaler object
scaler_file = os.path.join(expfolder, 'scaler.pkl')
pickle.dump(scaler, open(scaler_file, 'wb'))
# Fit model
print('\nFitting model...')
history_sed = []
f1s_best = 0
epoch_best = 0
epochs_since_improvement = 0
if resume:
f1s_best = resume_f1_best
for epoch in range(epochs):
if epoch_limit is None:
history = model.fit(x=x_train,
y=y_train,
batch_size=batch_size,
epochs=epoch + 1,
verbose=fit_verbose,
validation_split=0.0,
validation_data=(x_val, y_val),
shuffle=True,
initial_epoch=epoch)
# Test using sed_eval
pred = model.predict(x_val)
est_roll, ref_roll = combine_event_rolls(pred,
y_test,
threshold=0.5)
# ref_roll = y_val[:]
# est_roll = 1 * (pred >= 0.5)
est_event_list = event_roll_to_event_list(est_roll, label_list,
audio_hop / float(sr))
ref_event_list = event_roll_to_event_list(ref_roll, label_list,
audio_hop / float(sr))
seg_metrics1s = sed_eval.sound_event.SegmentBasedMetrics(
label_list, time_resolution=1.0)
seg_metrics1s.evaluate(ref_event_list, est_event_list)
results1s = seg_metrics1s.results()
seg_metrics100ms = sed_eval.sound_event.SegmentBasedMetrics(
label_list, time_resolution=0.1)
seg_metrics100ms.evaluate(ref_event_list, est_event_list)
results100ms = seg_metrics100ms.results()
scores = ('F1s: {:.4f}, P1s: {:.4f}, R1s: {:.4f}, F100ms: {:.4f}, '
'P100ms: {:.4f}, R100ms: {:.4f} | E1s: {:.4f}, '
'E100ms: {:.4f}'.format(
results1s['overall']['f_measure']['f_measure'],
results1s['overall']['f_measure']['precision'],
results1s['overall']['f_measure']['recall'],
results100ms['overall']['f_measure']['f_measure'],
results100ms['overall']['f_measure']['precision'],
results100ms['overall']['f_measure']['recall'],
results1s['overall']['error_rate']['error_rate'],
results100ms['overall']['error_rate']['error_rate']))
history_sed.append([results1s, results100ms])
print(scores)
# Save weights if we see improvement
f1s_current = results1s['overall']['f_measure']['f_measure']
improvement = False
if f1s_current > f1s_best:
improvement = True
f1s_best = f1s_current
weights_best_file = os.path.join(expfolder,
'weights_best.hdf5')
model.save_weights(weights_best_file)
# ********************* CHECKPOINT ************************
# Save history.history
history_score_file = os.path.join(expfolder,
'history_scores.json')
json.dump(history.history,
open(history_score_file, 'w'),
indent=2)
# Save history_sed
history_sed_file = os.path.join(expfolder, 'history_sed.json')
json.dump(history_sed, open(history_sed_file, 'w'), indent=2)
# Get predictions
pred = model.predict(x_test)
# Save Y_test, predictions and IDs
ytestfile = os.path.join(expfolder, 'ytest.npy.gz')
yprobfile = os.path.join(expfolder, 'yprob.npy.gz')
yidfile = os.path.join(expfolder, 'yid.npy.gz')
y_test.dump(gzip.open(ytestfile, 'wb'))
pred.dump(gzip.open(yprobfile, 'wb'))
id_test.dump(gzip.open(yidfile, 'wb'))
if improvement:
print(
'{:d} Best val F1s: {:.4f} (IMPROVEMENT, saving)\n'.format(
epoch, f1s_best))
epochs_since_improvement = 0
epoch_best = epoch
else:
print('{:d} Best val F1s: {:.4f} ({:d})\n'.format(
epoch, f1s_best, epoch_best))
epochs_since_improvement += 1
else:
order = np.arange(x_train.shape[0])
np.random.shuffle(order)
idx = 0
mini_epochs_per_epoch = int(
np.floor(
(x_train.shape[0] - epoch_limit) / float(epoch_limit))) + 1
mini_epoch = epoch * mini_epochs_per_epoch
while idx < x_train.shape[0] - epoch_limit:
history = model.fit(x=x_train[order[idx:idx + epoch_limit]],
y=y_train[order[idx:idx + epoch_limit]],
batch_size=batch_size,
epochs=mini_epoch + 1,
verbose=fit_verbose,
validation_split=0.0,
validation_data=(x_val, y_val),
shuffle=True,
initial_epoch=mini_epoch)
idx += epoch_limit
mini_epoch += 1
# Test using sed_eval
pred = model.predict(x_val)
est_roll, ref_roll = combine_event_rolls(pred,
y_test,
threshold=0.5)
# ref_roll = y_val[:]
# est_roll = 1 * (pred >= 0.5)
est_event_list = event_roll_to_event_list(
est_roll, label_list, audio_hop / float(sr))
ref_event_list = event_roll_to_event_list(
ref_roll, label_list, audio_hop / float(sr))
seg_metrics1s = sed_eval.sound_event.SegmentBasedMetrics(
label_list, time_resolution=1.0)
seg_metrics1s.evaluate(ref_event_list, est_event_list)
results1s = seg_metrics1s.results()
seg_metrics100ms = sed_eval.sound_event.SegmentBasedMetrics(
label_list, time_resolution=0.1)
seg_metrics100ms.evaluate(ref_event_list, est_event_list)
results100ms = seg_metrics100ms.results()
scores = (
'F1s: {:.4f}, P1s: {:.4f}, R1s: {:.4f}, F100ms: {:.4f}, '
'P100ms: {:.4f}, R100ms: {:.4f} | E1s: {:.4f}, '
'E100ms: {:.4f}'.format(
results1s['overall']['f_measure']['f_measure'],
results1s['overall']['f_measure']['precision'],
results1s['overall']['f_measure']['recall'],
results100ms['overall']['f_measure']['f_measure'],
results100ms['overall']['f_measure']['precision'],
results100ms['overall']['f_measure']['recall'],
results1s['overall']['error_rate']['error_rate'],
results100ms['overall']['error_rate']['error_rate']))
history_sed.append([results1s, results100ms])
print(scores)
# Save weights if we see improvement
f1s_current = results1s['overall']['f_measure']['f_measure']
improvement = False
if f1s_current > f1s_best:
improvement = True
f1s_best = f1s_current
weights_best_file = os.path.join(expfolder,
'weights_best.hdf5')
model.save_weights(weights_best_file)
# ********************* CHECKPOINT **********************
# Save history.history
history_score_file = os.path.join(expfolder,
'history_scores.json')
json.dump(history.history,
open(history_score_file, 'w'),
indent=2)
# Save history_sed
history_sed_file = os.path.join(expfolder,
'history_sed.json')
json.dump(history_sed,
open(history_sed_file, 'w'),
indent=2)
# Get predictions
pred = model.predict(x_test)
# Save Y_test, predictions and IDs
ytestfile = os.path.join(expfolder, 'ytest.npy.gz')
yprobfile = os.path.join(expfolder, 'yprob.npy.gz')
yidfile = os.path.join(expfolder, 'yid.npy.gz')
y_test.dump(gzip.open(ytestfile, 'wb'))
pred.dump(gzip.open(yprobfile, 'wb'))
id_test.dump(gzip.open(yidfile, 'wb'))
if improvement:
print('{:d} Best val F1s: {:.4f} (IMPROVEMENT, '
'saving)\n'.format(mini_epoch, f1s_best))
epochs_since_improvement = 0
epoch_best = mini_epoch
else:
print('{:d} Best val F1s: {:.4f} ({:d})\n'.format(
mini_epoch, f1s_best, epoch_best))
epochs_since_improvement += 1
if ((sed_early_stopping is not None)
and (epochs_since_improvement >= sed_early_stopping)):
print('\nNo improvement for {:d} epochs, stopping.'.format(
sed_early_stopping))
break
if ((sed_early_stopping is not None)
and (epochs_since_improvement >= sed_early_stopping)):
print('\nNo improvement for {:d} epochs, stopping.'.format(
sed_early_stopping))
break
# # Training curves
# plot_training_curves(history)
# Save model and predictions to disk
print('\nSaving model and predictions to disk...')
modeljsonfile = os.path.join(expfolder, 'model.json')
model_json = model.to_json()
with open(modeljsonfile, 'w') as json_file:
json.dump(model_json, json_file, indent=2)
# Save last version of weights (for resuming training)
weights_last_file = os.path.join(expfolder, 'weights_last.hdf5')
model.save_weights(weights_last_file)
# Save history.history
history_score_file = os.path.join(expfolder, 'history_scores.json')
json.dump(history.history, open(history_score_file, 'w'), indent=2)
# Save history_sed
history_sed_file = os.path.join(expfolder, 'history_sed.json')
json.dump(history_sed, open(history_sed_file, 'w'), indent=2)
# Save scaler object
scaler_file = os.path.join(expfolder, 'scaler.pkl')
pickle.dump(scaler, open(scaler_file, 'wb'))
# Save Y_test, predictions and IDs
ytestfile = os.path.join(expfolder, 'ytest.npy.gz')
yprobfile = os.path.join(expfolder, 'yprob.npy.gz')
yidfile = os.path.join(expfolder, 'yid.npy.gz')
y_test.dump(gzip.open(ytestfile, 'wb'))
id_test.dump(gzip.open(yidfile, 'wb'))
# Get predictions
model.load_weights(weights_best_file)
pred = model.predict(x_test)
pred.dump(gzip.open(yprobfile, 'wb'))
print("Done.")
def evaluate(expid, audio_hop=882, sr=44100, sequence_frames=128,
sequence_hop=64, normalize_data=True, mel_bands=40,
load_predictions=True, save_results=True):
fold_results = []
modelfolder = (
'/scratch/js7561/datasets/scaper_waspaa2017/urban-sed/models')
expfolder = os.path.join(modelfolder, expid)
if load_predictions:
ytestfile = os.path.join(expfolder, 'ytest.npy.gz')
yprobfile = os.path.join(expfolder, 'yprob.npy.gz')
yidfile = os.path.join(expfolder, 'yid.npy.gz')
y_test = np.load(gzip.open(ytestfile, 'rb'))
pred_test = np.load(gzip.open(yprobfile, 'rb'))
id_test = np.load(gzip.open(yidfile, 'rb'))
label_list = (['air_conditioner', 'car_horn', 'children_playing',
'dog_bark', 'drilling', 'engine_idling', 'gun_shot',
'jackhammer', 'siren', 'street_music'])
label_list = sorted(label_list)
else:
# Build model and load in best weights
model = build_crnn_onestep(n_freq_cnn=mel_bands)
model.compile(optimizer='adam', loss='binary_crossentropy',
metrics=['accuracy'])
weightfile = os.path.join(expfolder, 'weights_best.hdf5'.format(expid))
model.load_weights(weightfile)
# Load data
(x_train, y_train, id_train, x_val, y_val, id_val, x_test, y_test,
id_test, label_list, scaler) = load_urbansed_crnn(
sequence_frames=sequence_frames, sequence_hop=sequence_hop,
sequence_offset=0,
audio_hop=audio_hop, sr=sr,
verbose=False, normalize=normalize_data,
mel_bands=mel_bands)
pred_test = model.predict(x_test)
est_roll, ref_roll = combine_event_rolls(pred_test, y_test,
threshold=0.5)
# ref_roll = y_test[:]
# est_roll = 1 * (pred_test >= 0.5)
# COMPUTE FOLD RESULTS AND REPORT
# Convert event rolls into even lists
est_event_list = event_roll_to_event_list(
est_roll, label_list, audio_hop/float(sr))
ref_event_list = event_roll_to_event_list(
ref_roll, label_list, audio_hop/float(sr))
# Compute metrics at 1s, 100ms and 20ms levels
seg_metrics1s = sed_eval.sound_event.SegmentBasedMetrics(
label_list, time_resolution=1.0)
seg_metrics1s.evaluate(ref_event_list, est_event_list)
results1s = seg_metrics1s.results()
seg_metrics100ms = sed_eval.sound_event.SegmentBasedMetrics(
label_list, time_resolution=0.1)
seg_metrics100ms.evaluate(ref_event_list, est_event_list)
results100ms = seg_metrics100ms.results()
seg_metrics20ms = sed_eval.sound_event.SegmentBasedMetrics(
label_list, time_resolution=0.020)
seg_metrics20ms.evaluate(ref_event_list, est_event_list)
results20ms = seg_metrics20ms.results()
fold_results.append([results1s, results100ms, results20ms])
# Report scores
scores = (
'F1s: {:.4f}, P1s: {:.4f}, R1s: {:.4f}, F100ms: {:.4f}, '
'P100ms: {:.4f}, R100ms: {:.4f} | E1s: {:.4f}, '
'E100ms: {:.4f}'.format(
results1s['overall']['f_measure']['f_measure'],
results1s['overall']['f_measure']['precision'],
results1s['overall']['f_measure']['recall'],
results100ms['overall']['f_measure']['f_measure'],
results100ms['overall']['f_measure']['precision'],
results100ms['overall']['f_measure']['recall'],
results1s['overall']['error_rate']['error_rate'],
results100ms['overall']['error_rate']['error_rate']
))
print(scores)
# Save scores
results_all = {'results1s': results1s,
'results100ms': results100ms,
'results20ms': results20ms}
if save_results:
results_all_file = os.path.join(expfolder, 'test_results.json')
json.dump(results_all, open(results_all_file, 'w'), indent=2)
return results_all
def evaluate_load_test(expid,
audio_hop=882,
sr=44100,
sequence_frames=50,
sequence_hop=25,
normalize_data=True,
mel_bands=128,
large_cnn=False,
frame_level_y=False):
modelfolder = (
'/scratch/js7561/datasets/scaper_waspaa2017/urban-sed/models')
expfolder = os.path.join(modelfolder, expid)
# Build model and load in best weights
model = build_custom_cnn(n_freq_cnn=mel_bands, large_cnn=large_cnn)
model.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=['accuracy'])
weightfile = os.path.join(expfolder, 'weights_best.hdf5'.format(expid))
model.load_weights(weightfile)
# Load data
(x_test, y_test, id_test, label_list,
scaler) = load_urbansed_cnn_test(expid,
sequence_frames=sequence_frames,
audio_hop=audio_hop,
sr=sr,
verbose=False,
normalize=normalize_data,
mel_bands=mel_bands,
frame_level_y=frame_level_y,
load_subset=False,
file_list=None)
pred_test = model.predict(x_test)
if frame_level_y:
ref_roll = combine_event_rolls_single(y_test, threshold=None)
else:
ref_roll = y_test[:]
est_roll = 1 * (pred_test >= 0.5)
# COMPUTE FOLD RESULTS AND REPORT
# Convert event rolls into even lists
if frame_level_y:
est_event_list = event_roll_to_event_list(est_roll, label_list, 1.0)
ref_event_list = event_roll_to_event_list(ref_roll, label_list,
audio_hop / float(sr))
else:
est_event_list = event_roll_to_event_list(est_roll, label_list, 1.0)
ref_event_list = event_roll_to_event_list(ref_roll, label_list, 1.0)
# Compute metrics at 1s and 100ms levels
seg_metrics1s = sed_eval.sound_event.SegmentBasedMetrics(
label_list, time_resolution=1.0)
seg_metrics1s.evaluate(ref_event_list, est_event_list)
results1s = seg_metrics1s.results()
seg_metrics100ms = sed_eval.sound_event.SegmentBasedMetrics(
label_list, time_resolution=0.1)
seg_metrics100ms.evaluate(ref_event_list, est_event_list)
results100ms = seg_metrics100ms.results()
# Report scores
scores = ('F1s: {:.4f}, P1s: {:.4f}, R1s: {:.4f}, F100ms: {:.4f}, '
'P100ms: {:.4f}, R100ms: {:.4f} | E1s: {:.4f}, '
'E100ms: {:.4f}'.format(
results1s['overall']['f_measure']['f_measure'],
results1s['overall']['f_measure']['precision'],
results1s['overall']['f_measure']['recall'],
results100ms['overall']['f_measure']['f_measure'],
results100ms['overall']['f_measure']['precision'],
results100ms['overall']['f_measure']['recall'],
results1s['overall']['error_rate']['error_rate'],
results100ms['overall']['error_rate']['error_rate']))
print(scores)
# Save scores
results_all = {'results1s': results1s, 'results100ms': results100ms}
results_all_file = os.path.join(expfolder, 'test_results_load.json')
json.dump(results_all, open(results_all_file, 'w'), indent=2)
return results_all