def collect_ground_truth(data_gen, params):
data_in, data_out = data_gen.get_data_sizes()
gt = collect_test_labels(data_gen, data_out, params['mode'], params['quick_test'])
sed_gt = evaluation_metrics.reshape_3Dto2D(gt[0])
doa_gt = evaluation_metrics.reshape_3Dto2D(gt[1])
return sed_gt, doa_gt
def main(argv):
"""
Main wrapper for training sound event localization and detection network.
:param argv: expects two optional inputs.
first input: job_id - (optional) all the output files will be uniquely represented with this. (default) 1
second input: task_id - (optional) To chose the system configuration in parameters.py.
(default) uses default parameters
"""
if len(argv) != 3:
print('\n\n')
print('-------------------------------------------------------------------------------------------------------')
print('The code expected two inputs')
print('\t>> python seld.py <job-id> <task-id>')
print('\t\t<job-id> is a unique identifier which is used for output filenames (models, training plots). '
'You can use any number or string for this.')
print('\t\t<task-id> is used to choose the user-defined parameter set from parameter.py')
print('Using default inputs for now')
print('-------------------------------------------------------------------------------------------------------')
print('\n\n')
# use parameter set defined by user
task_id = '1' if len(argv) < 3 else argv[-1]
params = parameter.get_params(task_id)
job_id = 1 if len(argv) < 2 else argv[1]
model_dir = 'models/'
utils.create_folder(model_dir)
unique_name = '{}_ov{}_split{}_{}{}_3d{}_{}'.format(
params['dataset'], params['overlap'], params['split'], params['mode'], params['weakness'],
int(params['cnn_3d']), job_id
)
unique_name = os.path.join(model_dir, unique_name)
print("unique_name: {}\n".format(unique_name))
data_gen_train = cls_data_generator.DataGenerator(
dataset=params['dataset'], ov=params['overlap'], split=params['split'], db=params['db'], nfft=params['nfft'],
batch_size=params['batch_size'], seq_len=params['sequence_length'], classifier_mode=params['mode'],
weakness=params['weakness'], datagen_mode='train', cnn3d=params['cnn_3d'], xyz_def_zero=params['xyz_def_zero'],
azi_only=params['azi_only']
)
data_gen_test = cls_data_generator.DataGenerator(
dataset=params['dataset'], ov=params['overlap'], split=params['split'], db=params['db'], nfft=params['nfft'],
batch_size=params['batch_size'], seq_len=params['sequence_length'], classifier_mode=params['mode'],
weakness=params['weakness'], datagen_mode='test', cnn3d=params['cnn_3d'], xyz_def_zero=params['xyz_def_zero'],
azi_only=params['azi_only'], shuffle=False
)
data_in, data_out = data_gen_train.get_data_sizes()
print(
'FEATURES:\n'
'\tdata_in: {}\n'
'\tdata_out: {}\n'.format(
data_in, data_out
)
)
gt = collect_test_labels(data_gen_test, data_out, params['mode'], params['quick_test'])
sed_gt = evaluation_metrics.reshape_3Dto2D(gt[0])
doa_gt = evaluation_metrics.reshape_3Dto2D(gt[1])
print(
'MODEL:\n'
'\tdropout_rate: {}\n'
'\tCNN: nb_cnn_filt: {}, pool_size{}\n'
'\trnn_size: {}, fnn_size: {}\n'.format(
params['dropout_rate'],
params['nb_cnn3d_filt'] if params['cnn_3d'] else params['nb_cnn2d_filt'], params['pool_size'],
params['rnn_size'], params['fnn_size']
)
)
model = keras_model.get_model(data_in=data_in, data_out=data_out, dropout_rate=params['dropout_rate'],
nb_cnn2d_filt=params['nb_cnn2d_filt'], pool_size=params['pool_size'],
rnn_size=params['rnn_size'], fnn_size=params['fnn_size'],
classification_mode=params['mode'], weights=params['loss_weights'], loader=False, loader2=False) # Change loader to True to enable transfer learning, Change loader2 to True to enable transfer learning with different labels
best_metric = 99999
conf_mat = None
best_conf_mat = None
best_epoch = -1
patience_cnt = 0
epoch_metric_loss = np.zeros(params['nb_epochs'])
tr_loss = np.zeros(params['nb_epochs'])
val_loss = np.zeros(params['nb_epochs'])
doa_loss = np.zeros((params['nb_epochs'], 6))
sed_loss = np.zeros((params['nb_epochs'], 2))
nb_epoch = 2 if params['quick_test'] else params['nb_epochs']
for epoch_cnt in range(nb_epoch):
start = time.time()
hist = model.fit_generator(
generator=data_gen_train.generate(),
steps_per_epoch=2 if params['quick_test'] else data_gen_train.get_total_batches_in_data(),
validation_data=data_gen_test.generate(),
validation_steps=2 if params['quick_test'] else data_gen_test.get_total_batches_in_data(),
epochs=1,
verbose=0
)
tr_loss[epoch_cnt] = hist.history.get('loss')[-1]
val_loss[epoch_cnt] = hist.history.get('val_loss')[-1]
pred = model.predict_generator(
generator=data_gen_test.generate(),
steps=2 if params['quick_test'] else data_gen_test.get_total_batches_in_data(),
verbose=2
)
if params['mode'] == 'regr':
sed_pred = evaluation_metrics.reshape_3Dto2D(pred[0]) > 0.5
doa_pred = evaluation_metrics.reshape_3Dto2D(pred[1])
sed_loss[epoch_cnt, :] = evaluation_metrics.compute_sed_scores(sed_pred, sed_gt, data_gen_test.nb_frames_1s())
if params['azi_only']:
doa_loss[epoch_cnt, :], conf_mat = evaluation_metrics.compute_doa_scores_regr_xy(doa_pred, doa_gt,
sed_pred, sed_gt)
else:
doa_loss[epoch_cnt, :], conf_mat = evaluation_metrics.compute_doa_scores_regr_xyz(doa_pred, doa_gt,
sed_pred, sed_gt)
epoch_metric_loss[epoch_cnt] = np.mean([
sed_loss[epoch_cnt, 0],
1-sed_loss[epoch_cnt, 1],
2*np.arcsin(doa_loss[epoch_cnt, 1]/2.0)/np.pi,
1 - (doa_loss[epoch_cnt, 5] / float(doa_gt.shape[0]))]
)
plot_functions(unique_name, tr_loss, val_loss, sed_loss, doa_loss, epoch_metric_loss)
patience_cnt += 1
if epoch_metric_loss[epoch_cnt] < best_metric:
best_metric = epoch_metric_loss[epoch_cnt]
best_conf_mat = conf_mat
best_epoch = epoch_cnt
model.save('{}_model.h5'.format(unique_name))
patience_cnt = 0
print(
'epoch_cnt: %d, time: %.2fs, tr_loss: %.2f, val_loss: %.2f, '
'F1_overall: %.2f, ER_overall: %.2f, '
'doa_error_gt: %.2f, doa_error_pred: %.2f, good_pks_ratio:%.2f, '
'error_metric: %.2f, best_error_metric: %.2f, best_epoch : %d' %
(
epoch_cnt, time.time() - start, tr_loss[epoch_cnt], val_loss[epoch_cnt],
sed_loss[epoch_cnt, 1], sed_loss[epoch_cnt, 0],
doa_loss[epoch_cnt, 1], doa_loss[epoch_cnt, 2], doa_loss[epoch_cnt, 5] / float(sed_gt.shape[0]),
epoch_metric_loss[epoch_cnt], best_metric, best_epoch
)
)
if patience_cnt > params['patience']:
break
print('best_conf_mat : {}'.format(best_conf_mat))
print('best_conf_mat_diag : {}'.format(np.diag(best_conf_mat)))
print('saved model for the best_epoch: {} with best_metric: {}, '.format(best_epoch, best_metric))
print('DOA Metrics: doa_loss_gt: {}, doa_loss_pred: {}, good_pks_ratio: {}'.format(
doa_loss[best_epoch, 1], doa_loss[best_epoch, 2], doa_loss[best_epoch, 5] / float(sed_gt.shape[0])))
print('SED Metrics: ER_overall: {}, F1_overall: {}'.format(sed_loss[best_epoch, 1], sed_loss[best_epoch, 0]))
print('unique_name: {} '.format(unique_name))
def main(args):
"""
Main wrapper for training sound event localization and detection network.
:param argv: expects two optional inputs.
first input: job_id - (optional) all the output files will be uniquely represented with this. (default) 1
second input: task_id - (optional) To chose the system configuration in parameters.py.
(default) uses default parameters
"""
# use parameter set defined by user
task_id = args.params
params = parameter.get_params(task_id)
job_id = args.model_name
model_dir = 'models/' + args.author + '/' if args.author != "" else 'models/'
utils.create_folder(model_dir)
unique_name = '{}_ov{}_split{}_{}{}_3d{}_{}'.format(
params['dataset'], params['overlap'], params['split'], params['mode'],
params['weakness'], int(params['cnn_3d']), job_id)
model_name = unique_name
epoch_manager = JSON_Manager(args.author, unique_name)
logdir = "logs/" + args.author + "/" + unique_name
unique_name = os.path.join(model_dir, unique_name)
print("unique_name: {}\n".format(unique_name))
session = tf.InteractiveSession()
file_writer = tf.summary.FileWriter(logdir, session.graph)
data_gen_train = cls_data_generator.DataGenerator(
dataset=params['dataset'],
ov=params['overlap'],
split=params['split'],
db=params['db'],
nfft=params['nfft'],
batch_size=params['batch_size'],
seq_len=params['sequence_length'],
classifier_mode=params['mode'],
weakness=params['weakness'],
datagen_mode='train',
cnn3d=params['cnn_3d'],
xyz_def_zero=params['xyz_def_zero'],
azi_only=params['azi_only'])
data_gen_test = cls_data_generator.DataGenerator(
dataset=params['dataset'],
ov=params['overlap'],
split=params['split'],
db=params['db'],
nfft=params['nfft'],
batch_size=params['batch_size'],
seq_len=params['sequence_length'],
classifier_mode=params['mode'],
weakness=params['weakness'],
datagen_mode='test',
cnn3d=params['cnn_3d'],
xyz_def_zero=params['xyz_def_zero'],
azi_only=params['azi_only'],
shuffle=False)
data_in, data_out = data_gen_train.get_data_sizes()
print('FEATURES:\n'
'\tdata_in: {}\n'
'\tdata_out: {}\n'.format(data_in, data_out))
gt = collect_test_labels(data_gen_test, data_out, params['mode'],
params['quick_test'])
sed_gt = evaluation_metrics.reshape_3Dto2D(gt[0])
doa_gt = evaluation_metrics.reshape_3Dto2D(gt[1])
print('MODEL:\n'
'\tdropout_rate: {}\n'
'\tCNN: nb_cnn_filt: {}, pool_size{}\n'
'\trnn_size: {}, fnn_size: {}\n'.format(
params['dropout_rate'], params['nb_cnn3d_filt']
if params['cnn_3d'] else params['nb_cnn2d_filt'],
params['pool_size'], params['rnn_size'], params['fnn_size']))
model = keras_model.get_model(data_in=data_in,
data_out=data_out,
dropout_rate=params['dropout_rate'],
nb_cnn2d_filt=params['nb_cnn2d_filt'],
pool_size=params['pool_size'],
rnn_size=params['rnn_size'],
fnn_size=params['fnn_size'],
classification_mode=params['mode'],
weights=params['loss_weights'])
initial = epoch_manager.get_epoch()
if initial != 0:
print(f"Resume training from epoch {initial}")
print("Loading already trained model...")
# In order to load custom layers we need to link the references to the custom objects
model = load_model(os.path.join(model_dir, model_name + "_model.h5"),
custom_objects={
'QuaternionConv2D': QuaternionConv2D,
'QuaternionGRU': QuaternionGRU,
'QuaternionDense': QuaternionDense
})
best_metric = epoch_manager.get_best_metric()
best_std = epoch_manager.get_best_std()
conf_mat = None
best_conf_mat = epoch_manager.get_best_conf_mat()
best_epoch = epoch_manager.get_best_epoch()
patience_cnt = epoch_manager.get_patience_cnt()
epoch_metric_loss = np.zeros(params['nb_epochs'])
sed_score = np.zeros(params['nb_epochs'])
std_score = np.zeros(params['nb_epochs'])
doa_score = np.zeros(params['nb_epochs'])
seld_score = np.zeros(params['nb_epochs'])
tr_loss = np.zeros(params['nb_epochs'])
val_loss = np.zeros(params['nb_epochs'])
doa_loss = np.zeros((params['nb_epochs'], 6))
sed_loss = np.zeros((params['nb_epochs'], 2))
time_hold = tf.placeholder(tf.float32, shape=None, name='time_summary')
time_summ = tf.summary.scalar('time', time_hold)
tr_loss_hold = tf.placeholder(tf.float32,
shape=None,
name='tr_loss_summary')
tr_loss_summ = tf.summary.scalar('tr_loss', tr_loss_hold)
val_loss_hold = tf.placeholder(tf.float32,
shape=None,
name='val_loss_summary')
val_loss_summ = tf.summary.scalar('val_loss', val_loss_hold)
f1_hold = tf.placeholder(tf.float32, shape=None, name='f1_summary')
f1_summ = tf.summary.scalar('F1_overall', f1_hold)
er_hold = tf.placeholder(tf.float32, shape=None, name='er_summary')
er_summ = tf.summary.scalar('ER_overall', er_hold)
doa_error_gt_hold = tf.placeholder(tf.float32,
shape=None,
name='doa_error_gt_summary')
doa_error_gt_summ = tf.summary.scalar('doa_error_gt', doa_error_gt_hold)
doa_error_pred_hold = tf.placeholder(tf.float32,
shape=None,
name='doa_error_pred_summary')
doa_error_pred_summ = tf.summary.scalar('doa_error_pred',
doa_error_pred_hold)
good_pks_hold = tf.placeholder(tf.float32,
shape=None,
name='good_pks_summary')
good_pks_summ = tf.summary.scalar('good_pks_ratio', good_pks_hold)
sed_score_hold = tf.placeholder(tf.float32,
shape=None,
name='sed_score_summary')
sed_score_summ = tf.summary.scalar('sed_score', sed_score_hold)
doa_score_hold = tf.placeholder(tf.float32,
shape=None,
name='doa_score_summary')
doa_score_summ = tf.summary.scalar('doa_score', doa_score_hold)
seld_score_hold = tf.placeholder(tf.float32,
shape=None,
name='seld_score_summary')
seld_score_summ = tf.summary.scalar('seld_score', seld_score_hold)
std_score_hold = tf.placeholder(tf.float32,
shape=None,
name='std_score_summary')
std_score_summ = tf.summary.scalar('std_score', std_score_hold)
best_error_metric_hold = tf.placeholder(tf.float32,
shape=None,
name='best_error_metric_summary')
best_error_metric_summ = tf.summary.scalar('best_error_metric',
best_error_metric_hold)
best_epoch_hold = tf.placeholder(tf.float32,
shape=None,
name='best_epoch_summary')
best_epoch_summ = tf.summary.scalar('best_epoch', best_epoch_hold)
best_std_hold = tf.placeholder(tf.float32,
shape=None,
name='best_std_summary')
best_std_summ = tf.summary.scalar('best_std', best_std_hold)
merged = tf.summary.merge_all()
for epoch_cnt in range(initial, params['nb_epochs']):
start = time.time()
hist = model.fit_generator(
generator=data_gen_train.generate(),
steps_per_epoch=5 if params['quick_test'] else
data_gen_train.get_total_batches_in_data(),
validation_data=data_gen_test.generate(),
validation_steps=5 if params['quick_test'] else
data_gen_test.get_total_batches_in_data(),
use_multiprocessing=False,
epochs=1,
verbose=1)
tr_loss[epoch_cnt] = hist.history.get('loss')[-1]
val_loss[epoch_cnt] = hist.history.get('val_loss')[-1]
pred = model.predict_generator(
generator=data_gen_test.generate(),
steps=5 if params['quick_test'] else
data_gen_test.get_total_batches_in_data(),
use_multiprocessing=False,
verbose=2)
print("pred:", pred[1].shape)
if params['mode'] == 'regr':
sed_pred = evaluation_metrics.reshape_3Dto2D(pred[0]) > 0.5
doa_pred = evaluation_metrics.reshape_3Dto2D(pred[1])
sed_loss[epoch_cnt, :] = evaluation_metrics.compute_sed_scores(
sed_pred, sed_gt, data_gen_test.nb_frames_1s())
if params['azi_only']:
doa_loss[
epoch_cnt, :], conf_mat = evaluation_metrics.compute_doa_scores_regr_xy(
doa_pred, doa_gt, sed_pred, sed_gt)
else:
doa_loss[
epoch_cnt, :], conf_mat = evaluation_metrics.compute_doa_scores_regr_xyz(
doa_pred, doa_gt, sed_pred, sed_gt)
sed_score[epoch_cnt] = np.mean(
[sed_loss[epoch_cnt, 0], 1 - sed_loss[epoch_cnt, 1]])
doa_score[epoch_cnt] = np.mean([
2 * np.arcsin(doa_loss[epoch_cnt, 1] / 2.0) / np.pi,
1 - (doa_loss[epoch_cnt, 5] / float(doa_gt.shape[0]))
])
seld_score[epoch_cnt] = np.mean(
[sed_score[epoch_cnt], doa_score[epoch_cnt]])
# standard deviation
std_score[epoch_cnt] = np.std(
[sed_score[epoch_cnt], doa_score[epoch_cnt]])
plot_functions(unique_name, tr_loss, val_loss, sed_loss, doa_loss,
sed_score, doa_score)
patience_cnt += 1
epoch_manager.increase_patience_cnt()
model.save('{}_model.h5'.format(unique_name))
if seld_score[epoch_cnt] < best_metric:
best_metric = seld_score[epoch_cnt]
epoch_manager.set_best_metric(best_metric)
best_std = std_score[epoch_cnt]
epoch_manager.set_best_std(best_std)
best_conf_mat = conf_mat
epoch_manager.set_best_conf_mat(conf_mat)
best_epoch = epoch_cnt
epoch_manager.set_best_epoch(best_epoch)
model.save('{}_best_model.h5'.format(unique_name))
patience_cnt = 0
epoch_manager.reset_patience_cnt()
if patience_cnt > params['patience']:
print(
f"\n---- PATIENCE TRIGGERED AFTER {epoch_cnt} EPOCHS ----\n")
break
summary = session.run(merged,
feed_dict={
time_hold:
time.time() - start,
tr_loss_hold:
tr_loss[epoch_cnt],
val_loss_hold:
val_loss[epoch_cnt],
f1_hold:
sed_loss[epoch_cnt, 1],
er_hold:
sed_loss[epoch_cnt, 0],
doa_error_gt_hold:
doa_loss[epoch_cnt, 1],
doa_error_pred_hold:
doa_loss[epoch_cnt, 2],
good_pks_hold:
doa_loss[epoch_cnt, 5] /
float(sed_gt.shape[0]),
sed_score_hold:
sed_score[epoch_cnt],
doa_score_hold:
doa_score[epoch_cnt],
seld_score_hold:
seld_score[epoch_cnt],
std_score_hold:
std_score[epoch_cnt],
best_error_metric_hold:
best_metric,
best_epoch_hold:
best_epoch,
best_std_hold:
best_std
})
file_writer.add_summary(summary, epoch_cnt)
print(
'epoch_cnt: %d, time: %.2fs, tr_loss: %.2f, val_loss: %.2f, '
'F1_overall: %.2f, ER_overall: %.2f, '
'doa_error_gt: %.2f, doa_error_pred: %.2f, good_pks_ratio:%.2f, '
'sed_score: %.2f, doa_score: %.2f, best_error_metric: %.2f, best_epoch : %d, best_std: %.2f'
% (epoch_cnt, time.time() - start, tr_loss[epoch_cnt],
val_loss[epoch_cnt], sed_loss[epoch_cnt, 1],
sed_loss[epoch_cnt, 0], doa_loss[epoch_cnt, 1],
doa_loss[epoch_cnt, 2], doa_loss[epoch_cnt, 5] /
float(sed_gt.shape[0]), sed_score[epoch_cnt],
doa_score[epoch_cnt], best_metric, best_epoch, best_std))
epoch_manager.increase_epoch()
lower_confidence, upper_confidence = evaluation_metrics.compute_confidence_interval(
best_metric, best_std, params['nb_epochs'],
confid_coeff=1.96) # 1.96 for a 95% CI
print("\n---- FINISHED TRAINING ----\n")
print('best_conf_mat : {}'.format(best_conf_mat))
print('best_conf_mat_diag : {}'.format(np.diag(best_conf_mat)))
print('saved model for the best_epoch: {} with best_metric: {}, '.format(
best_epoch, best_metric))
print(
'DOA Metrics: doa_loss_gt: {}, doa_loss_pred: {}, good_pks_ratio: {}'.
format(doa_loss[best_epoch, 1], doa_loss[best_epoch, 2],
doa_loss[best_epoch, 5] / float(sed_gt.shape[0])))
print('SED Metrics: ER_overall: {}, F1_overall: {}'.format(
sed_loss[best_epoch, 0], sed_loss[best_epoch, 1]))
print('Confidence Interval: lower_interval: {}, upper_inteval: {}'.format(
lower_confidence, upper_confidence))
print('unique_name: {} '.format(unique_name))
label_sed = label[0].cuda()
label_doa = label[1].cuda()
for i_train in range(100):
X = input_.clone()
X = rotationLayer(X)
y_sed, y_doa = net(X, update=False)
y_doa = rotationLayer(y_doa)
optimizer.zero_grad()
loss_sed = criterion_sed(y_sed, label_sed)
loss_doa = unit_vec_distance(y_doa, label_doa, label_sed)
# loss = -loss_sed - loss_doa
# loss = - loss_doa
loss = -loss_sed
if i == 0:
sed_est = reshape_3Dto2D(
torch.sigmoid(y_sed).cpu().detach().numpy())
if highest_loss < loss_sed.item() + loss_doa.item():
highest_loss = loss_sed.item() + loss_doa.item()
ax_t = np.arange(sed_est.shape[0]) * 0.02
for j, v in enumerate(sed_est.T):
plt.plot(ax_t, v + j)
plt.xlabel('Time [sec]')
plt.ylabel('On / Off')
plt.tight_layout()
plt.savefig(
os.path.join(dirpath,
'sed_est_' + str(i_train + 1) + '.png'))
plt.close()
if i_train == 0:
sed_gt = reshape_3Dto2D(label[0].numpy())
for j, v in enumerate(sed_gt.T):
def main(argv):
"""
Main wrapper for training sound event localization and detection network.
:param argv: expects two optional inputs.
first input: job_id - (optional) all the output files will be uniquely represented with this. (default) 1
second input: task_id - (optional) To chose the system configuration in parameters.py.
(default) uses default parameters
"""
if len(argv) != 3:
print('\n\n')
print(
'-------------------------------------------------------------------------------------------------------'
)
print('The code expected two inputs')
print('\t>> python seld.py <job-id> <task-id>')
print(
'\t\t<job-id> is a unique identifier which is used for output filenames (models, training plots). '
'You can use any number or string for this.')
print(
'\t\t<task-id> is used to choose the user-defined parameter set from parameter.py'
)
print('Using default inputs for now')
print(
'-------------------------------------------------------------------------------------------------------'
)
print('\n\n')
# use parameter set defined by user
task_id = '1' if len(argv) < 3 else argv[-1]
params = parameter.get_params(task_id)
job_id = 1 if len(argv) < 2 else argv[1]
model_dir = 'models/'
utils.create_folder(model_dir)
unique_name = '{}_ov{}_split{}_{}{}_3d{}_{}'.format(
params['dataset'], params['overlap'], params['split'], params['mode'],
params['weakness'], int(params['cnn_3d']), job_id)
unique_name = os.path.join(model_dir, unique_name)
print("unique_name: {}\n".format(unique_name))
data_gen_train = cls_data_generator.DataGenerator(
dataset=params['dataset'],
ov=params['overlap'],
split=params['split'],
db=params['db'],
nfft=params['nfft'],
batch_size=params['batch_size'],
seq_len=params['sequence_length'],
classifier_mode=params['mode'],
weakness=params['weakness'],
datagen_mode='train',
cnn3d=params['cnn_3d'],
xyz_def_zero=params['xyz_def_zero'],
azi_only=params['azi_only'])
data_gen_test = cls_data_generator.DataGenerator(
dataset=params['dataset'],
ov=params['overlap'],
split=params['split'],
db=params['db'],
nfft=params['nfft'],
batch_size=params['batch_size'],
seq_len=params['sequence_length'],
classifier_mode=params['mode'],
weakness=params['weakness'],
datagen_mode='test',
cnn3d=params['cnn_3d'],
xyz_def_zero=params['xyz_def_zero'],
azi_only=params['azi_only'],
shuffle=False)
data_in, data_out = data_gen_train.get_data_sizes()
print('FEATURES:\n'
'\tdata_in: {}\n'
'\tdata_out: {}\n'.format(data_in, data_out))
gt = collect_test_labels(data_gen_test, data_out, params['mode'],
params['quick_test'])
sed_gt = evaluation_metrics.reshape_3Dto2D(gt[0])
doa_gt = evaluation_metrics.reshape_3Dto2D(gt[1])
print('MODEL:\n'
'\tdropout_rate: {}\n'
'\tCNN: nb_cnn_filt: {}, pool_size{}\n'
'\trnn_size: {}, fnn_size: {}\n'.format(
params['dropout_rate'], params['nb_cnn3d_filt']
if params['cnn_3d'] else params['nb_cnn2d_filt'],
params['pool_size'], params['rnn_size'], params['fnn_size']))
# TPU CODE FOR GOOGLE COLABORATORY
resolver = tf.distribute.cluster_resolver.TPUClusterResolver(
tpu='grpc://' + os.environ['COLAB_TPU_ADDR'])
tf.config.experimental_connect_to_cluster(resolver)
# This is the TPU initialization code that has to be at the beginning.
tf.tpu.experimental.initialize_tpu_system(resolver)
print("All devices: ", tf.config.list_logical_devices('TPU'))
strategy = tf.distribute.experimental.TPUStrategy(resolver)
with strategy.scope():
# Load or create model
model = utils.load_model(unique_name)
if model is None:
model = keras_model.get_model(
data_in=data_in,
data_out=data_out,
dropout_rate=params['dropout_rate'],
nb_cnn2d_filt=params['nb_cnn2d_filt'],
pool_size=params['pool_size'],
rnn_size=params['rnn_size'],
fnn_size=params['fnn_size'],
classification_mode=params['mode'],
weights=params['loss_weights'])
model.summary()
best_metric = 99999
conf_mat = None
best_conf_mat = None
best_epoch = -1
patience_cnt = 0
epoch_metric_loss = np.zeros(params['nb_epochs'])
sed_score = np.zeros(params['nb_epochs'])
doa_score = np.zeros(params['nb_epochs'])
tr_loss = np.zeros(params['nb_epochs'])
val_loss = np.zeros(params['nb_epochs'])
doa_loss = np.zeros((params['nb_epochs'], 6))
sed_loss = np.zeros((params['nb_epochs'], 2))
for epoch_cnt in range(params['nb_epochs']):
start = time.time()
hist = model.fit_generator(
generator=data_gen_train.generate(),
steps_per_epoch=5 if params['quick_test'] else
data_gen_train.get_total_batches_in_data(),
validation_data=data_gen_test.generate(),
validation_steps=5 if params['quick_test'] else
data_gen_test.get_total_batches_in_data(),
use_multiprocessing=False,
epochs=1,
verbose=1)
tr_loss[epoch_cnt] = hist.history.get('loss')[-1]
val_loss[epoch_cnt] = hist.history.get('val_loss')[-1]
pred = model.predict_generator(
generator=data_gen_test.generate(),
steps=5 if params['quick_test'] else
data_gen_test.get_total_batches_in_data(),
use_multiprocessing=False,
verbose=2)
print("pred:", pred[1].shape)
if params['mode'] == 'regr':
sed_pred = evaluation_metrics.reshape_3Dto2D(pred[0]) > 0.5
doa_pred = evaluation_metrics.reshape_3Dto2D(pred[1])
sed_loss[epoch_cnt, :] = evaluation_metrics.compute_sed_scores(
sed_pred, sed_gt, data_gen_test.nb_frames_1s())
if params['azi_only']:
doa_loss[
epoch_cnt, :], conf_mat = evaluation_metrics.compute_doa_scores_regr_xy(
doa_pred, doa_gt, sed_pred, sed_gt)
else:
doa_loss[
epoch_cnt, :], conf_mat = evaluation_metrics.compute_doa_scores_regr_xyz(
doa_pred, doa_gt, sed_pred, sed_gt)
# epoch_metric_loss[epoch_cnt] = np.mean([
# sed_loss[epoch_cnt, 0],
# 1-sed_loss[epoch_cnt, 1],
# 2*np.arcsin(doa_loss[epoch_cnt, 1]/2.0)/np.pi,
# 1 - (doa_loss[epoch_cnt, 5] / float(doa_gt.shape[0]))]
# )
sed_score[epoch_cnt] = np.mean(
[sed_loss[epoch_cnt, 0], 1 - sed_loss[epoch_cnt, 1]])
doa_score[epoch_cnt] = np.mean([
2 * np.arcsin(doa_loss[epoch_cnt, 1] / 2.0) / np.pi,
1 - (doa_loss[epoch_cnt, 5] / float(doa_gt.shape[0]))
])
#plot_functions(unique_name, tr_loss, val_loss, sed_loss, doa_loss, epoch_metric_loss)
plot_functions(unique_name, tr_loss, val_loss, sed_loss, doa_loss,
sed_score, doa_score)
patience_cnt += 1
# if epoch_metric_loss[epoch_cnt] < best_metric:
# best_metric = epoch_metric_loss[epoch_cnt]
# best_conf_mat = conf_mat
# best_epoch = epoch_cnt
# model.save('{}_model.h5'.format(unique_name))
# patience_cnt = 0
if sed_score[epoch_cnt] < best_metric:
best_metric = sed_score[epoch_cnt]
best_conf_mat = conf_mat
best_epoch = epoch_cnt
model.save('{}_model.h5'.format(unique_name))
patience_cnt = 0
print(
'epoch_cnt: %d, time: %.2fs, tr_loss: %.2f, val_loss: %.2f, '
'F1_overall: %.2f, ER_overall: %.2f, '
'doa_error_gt: %.2f, doa_error_pred: %.2f, good_pks_ratio:%.2f, '
'sed_score: %.2f, doa_score: %.2f, best_error_metric: %.2f, best_epoch : %d'
% (epoch_cnt, time.time() - start, tr_loss[epoch_cnt],
val_loss[epoch_cnt], sed_loss[epoch_cnt, 1],
sed_loss[epoch_cnt, 0], doa_loss[epoch_cnt, 1],
doa_loss[epoch_cnt, 2], doa_loss[epoch_cnt, 5] /
float(sed_gt.shape[0]), sed_score[epoch_cnt],
doa_score[epoch_cnt], best_metric, best_epoch))
#plot_functions(unique_name, tr_loss, val_loss, sed_loss, doa_loss, sed_score, doa_score, epoch_cnt)
print('best_conf_mat : {}'.format(best_conf_mat))
print('best_conf_mat_diag : {}'.format(np.diag(best_conf_mat)))
print('saved model for the best_epoch: {} with best_metric: {}, '.format(
best_epoch, best_metric))
print(
'DOA Metrics: doa_loss_gt: {}, doa_loss_pred: {}, good_pks_ratio: {}'.
format(doa_loss[best_epoch, 1], doa_loss[best_epoch, 2],
doa_loss[best_epoch, 5] / float(sed_gt.shape[0])))
print('SED Metrics: ER_overall: {}, F1_overall: {}'.format(
sed_loss[best_epoch, 0], sed_loss[best_epoch, 1]))
print('unique_name: {} '.format(unique_name))
def main(argv):
"""
Main wrapper for training sound event localization and detection network.
:param argv: expects two optional inputs.
first input: job_id - (optional) all the output files will be uniquely represented with this. (default) 1
second input: task_id - (optional) To chose the system configuration in parameters.py.
(default) uses default parameters
"""
if len(argv) != 3:
print('\n\n')
print('-------------------------------------------------------------------------------------------------------')
print('The code expected two inputs')
print('\t>> python seld.py <job-id> <task-id>')
print('\t\t<job-id> is a unique identifier which is used for output filenames (models, training plots). '
'You can use any number or string for this.')
print('\t\t<task-id> is used to choose the user-defined parameter set from parameter.py')
print('Using default inputs for now')
print('-------------------------------------------------------------------------------------------------------')
print('\n\n')
# use parameter set defined by user
task_id = '1' if len(argv) < 3 else argv[-1]
params = parameter.get_params(task_id)
job_id = 1 if len(argv) < 2 else argv[1]
model_dir = 'models/'
utils.create_folder(model_dir)
unique_name = '{}_train{}_validation{}_seq{}'.format(params['dataset'], params['train_split'], params['val_split'], params['sequence_length'])
unique_name = os.path.join(model_dir, unique_name)
print("unique_name: {}\n".format(unique_name))
# Cycling over overlaps
for ov in range(1, params['overlap']+1):
data_gen_test = cls_data_generator.DataGenerator(
dataset=params['dataset'], ov=params['overlap'], ov_num=ov, split=params['test_split'], db=params['db'], nfft=params['nfft'],
batch_size=params['batch_size'], seq_len=params['sequence_length'], classifier_mode=params['mode'],
weakness=params['weakness'], datagen_mode='test', cnn3d=params['cnn_3d'], xyz_def_zero=params['xyz_def_zero'],
azi_only=params['azi_only'], shuffle=False
)
data_in, data_out = data_gen_test.get_data_sizes()
n_classes = data_out[0][2]
print(
'FEATURES:\n'
'\tdata_in: {}\n'
'\tdata_out: {}\n'.format(
data_in, data_out
)
)
gt = collect_test_labels(data_gen_test, data_out, params['mode'], params['quick_test'])
sed_gt = evaluation_metrics.reshape_3Dto2D(gt[0])
doa_gt = evaluation_metrics.reshape_3Dto2D(gt[1])
print("#### Saving DOA and SED GT Values ####")
f = open("models/doa_gt.txt", "w+")
for elem in doa_gt:
f.write(str(list(elem)) + "\n")
f.close()
f = open("models/sed_gt.txt", "w+")
for elem in sed_gt:
f.write(str(elem)+"\n")
f.close()
print("######################################")
print(
'MODEL:\n'
'\tdropout_rate: {}\n'
'\tCNN: nb_cnn_filt: {}, pool_size{}\n'
'\trnn_size: {}, fnn_size: {}\n'.format(
params['dropout_rate'],
params['nb_cnn3d_filt'] if params['cnn_3d'] else params['nb_cnn2d_filt'], params['pool_size'],
params['rnn_size'], params['fnn_size']
)
)
model = keras_model.get_model(data_in=data_in, data_out=data_out, dropout_rate=params['dropout_rate'],
nb_cnn2d_filt=params['nb_cnn2d_filt'], pool_size=params['pool_size'],
rnn_size=params['rnn_size'], fnn_size=params['fnn_size'],
classification_mode=params['mode'], weights=params['loss_weights'], summary=False)
if(os.path.exists('{}_model.ckpt'.format(unique_name))):
print("Model found!")
model.load_weights('{}_model.ckpt'.format(unique_name))
for i in range(10):
print("###")
sed_score = np.zeros(params['nb_epochs'])
doa_score = np.zeros(params['nb_epochs'])
seld_score = np.zeros(params['nb_epochs'])
tr_loss = np.zeros(params['nb_epochs'])
val_loss = np.zeros(params['nb_epochs'])
doa_loss = np.zeros((params['nb_epochs'], 6))
sed_loss = np.zeros((params['nb_epochs'], 2))
epoch_cnt = 0
start = time.time()
print("#### Prediction on validation split ####")
pred = model.predict_generator(
generator=data_gen_test.generate(),
steps=params['quick_test_steps'] if params['quick_test'] else data_gen_test.get_total_batches_in_data(),
use_multiprocessing=False,
workers=1,
verbose=1,
)
print("##########################")
#print("pred:", pred[1].shape)
if params['mode'] == 'regr':
sed_pred = np.array(evaluation_metrics.reshape_3Dto2D(pred[0])) > .5
doa_pred = evaluation_metrics.reshape_3Dto2D(pred[1])
print("#### Saving DOA and SED Pred Values ####")
f = open("models/doa_pred.txt", "w+")
for elem in doa_pred:
f.write(str(list(elem)) + "\n")
f.close()
f = open("models/sed_pred.txt", "w+")
for elem in sed_pred:
f.write(str(elem)+"\n")
f.close()
print("########################################")
# Old version of confidence intervals
'''
# Computing confidence intervals
sed_err = sed_gt - sed_pred
[sed_conf_low, sed_conf_up, sed_median] = compute_confidence(sed_err)
# print("Condidence Interval for SED error is [" + str(sed_conf_low) + ", " + str(sed_conf_up) + "]")
print("Confidence Interval for SED error is [ %f, %f ]" % (sed_conf_low, sed_conf_up))
# print("\tMedian is " + str(sed_median))
print("\tMedian is %f" % (sed_median))
# print("\tDisplacement: +/- " + str(sed_conf_up - sed_median))
print("\tDisplacement: +/- %f" % (sed_conf_up - sed_median))
doa_err = doa_gt - doa_pred
[doa_conf_low, doa_conf_up, doa_median] = compute_confidence(doa_err)
# print("Condidence Interval for DOA is [" + str(doa_conf_low) + ", " + str(doa_conf_up) + "]")
print("Confidence Interval for DOA is [ %f, %f ]" % (doa_conf_low, doa_conf_up))
# print("Median is " + str(doa_median))
print("\tMedian is %f" % (doa_median))
# print("Displacement: +/- " + str(doa_conf_up - doa_median))
print("\tDisplacement: +/- %f" % (doa_conf_up - doa_median))
# ------------------------------
'''
sed_loss[epoch_cnt, :] = evaluation_metrics.compute_sed_scores(sed_pred, sed_gt, data_gen_test.nb_frames_1s())
if params['azi_only']:
doa_loss[epoch_cnt, :], conf_mat = evaluation_metrics.compute_doa_scores_regr_xy(doa_pred, doa_gt,
sed_pred, sed_gt)
else:
doa_loss[epoch_cnt, :], conf_mat = evaluation_metrics.compute_doa_scores_regr_xyz(doa_pred, doa_gt,
sed_pred, sed_gt)
sed_score[epoch_cnt] = np.mean([sed_loss[epoch_cnt, 0], 1 - sed_loss[epoch_cnt, 1]])
doa_score[epoch_cnt] = np.mean([2 * np.arcsin(doa_loss[epoch_cnt, 1] / 2.0) / np.pi,
1 - (doa_loss[epoch_cnt, 5] / float(doa_gt.shape[0]))])
seld_score[epoch_cnt] = (sed_score[epoch_cnt] + doa_score[epoch_cnt]) / 2
if os.path.isdir('./models'):
plot.imshow(conf_mat, cmap='binary', interpolation='None')
plot.savefig('models/confusion_matrix.jpg')
# New confidence computation, differing doa and sed errors
sed_err = sed_loss[epoch_cnt, 0]
[sed_conf_low, sed_conf_up] = compute_confidence(sed_err, sed_pred.shape[0])
print("Confidence Interval for SED error is [ %f, %f ]" % (sed_conf_low, sed_conf_up))
doa_err = doa_gt - doa_pred
[x_err, y_err, z_err] = compute_doa_confidence(doa_err, n_classes)
print('epoch_cnt: %d, time: %.2fs, tr_loss: %.4f, val_loss: %.4f, '
'F1_overall: %.2f, ER_overall: %.2f, '
'doa_error_gt: %.2f, doa_error_pred: %.2f, good_pks_ratio:%.2f, '
'sed_score: %.4f, doa_score: %.4f, seld_score: %.4f' %
(
epoch_cnt, time.time() - start, tr_loss[epoch_cnt], val_loss[epoch_cnt],
sed_loss[epoch_cnt, 1], sed_loss[epoch_cnt, 0],
doa_loss[epoch_cnt, 1], doa_loss[epoch_cnt, 2], doa_loss[epoch_cnt, 5] / float(sed_gt.shape[0]),
sed_score[epoch_cnt], doa_score[epoch_cnt], seld_score[epoch_cnt]
)
)
simple_plotter.plot_3d("models/doa_gt.txt", "models/doa_pred.txt", 0, 11, 200)
simple_plotter.plot_confidence(x_err, y_err, z_err, "ov"+str(ov))
def main(argv):
"""
Main wrapper for training sound event localization and detection network.
:param argv: expects two optional inputs.
first input: job_id - (optional) all the output files will be uniquely represented with this. (default) 1
second input: task_id - (optional) To chose the system configuration in parameters.py.
(default) uses default parameters
"""
if len(argv) != 3:
print('\n\n')
print('-------------------------------------------------------------------------------------------------------')
print('The code expected two inputs')
print('\t>> python seld.py <job-id> <task-id>')
print('\t\t<job-id> is a unique identifier which is used for output filenames (models, training plots). '
'You can use any number or string for this.')
print('\t\t<task-id> is used to choose the user-defined parameter set from parameter.py')
print('Using default inputs for now')
print('-------------------------------------------------------------------------------------------------------')
print('\n\n')
# use parameter set defined by user
task_id = '1' if len(argv) < 3 else argv[-1]
params = parameter.get_params(task_id)
job_id = 1 if len(argv) < 2 else argv[1]
model_dir = 'models/'
utils.create_folder(model_dir)
unique_name = '{}_train{}_validation{}_seq{}'.format(params['dataset'], params['train_split'], params['val_split'],
params['sequence_length'])
unique_name = os.path.join(model_dir, unique_name)
print("unique_name: {}\n".format(unique_name))
data_gen_train = cls_data_generator.DataGenerator(
dataset=params['dataset'], ov=params['overlap'], split=params['train_split'], db=params['db'],
nfft=params['nfft'],
batch_size=params['batch_size'], seq_len=params['sequence_length'], classifier_mode=params['mode'],
weakness=params['weakness'], datagen_mode='train', cnn3d=params['cnn_3d'], xyz_def_zero=params['xyz_def_zero'],
azi_only=params['azi_only']
)
data_gen_test = cls_data_generator.DataGenerator(
dataset=params['dataset'], ov=params['overlap'], split=params['val_split'], db=params['db'],
nfft=params['nfft'],
batch_size=params['batch_size'], seq_len=params['sequence_length'], classifier_mode=params['mode'],
weakness=params['weakness'], datagen_mode='validation', cnn3d=params['cnn_3d'], xyz_def_zero=params['xyz_def_zero'],
azi_only=params['azi_only'], shuffle=False
)
data_in, data_out = data_gen_train.get_data_sizes()
#n_classes = data_out[0][2]
print(
'FEATURES:\n'
'\tdata_in: {}\n'
'\tdata_out: {}\n'.format(
data_in, data_out
)
)
gt = collect_test_labels(data_gen_test, data_out, params['mode'], params['quick_test'])
sed_gt = evaluation_metrics.reshape_3Dto2D(gt[0])
doa_gt = evaluation_metrics.reshape_3Dto2D(gt[1])
print(
'MODEL:\n'
'\tdropout_rate: {}\n'
'\tCNN: nb_cnn_filt: {}, pool_size{}\n'
'\trnn_size: {}, fnn_size: {}\n'.format(
params['dropout_rate'],
params['nb_cnn3d_filt'] if params['cnn_3d'] else params['nb_cnn2d_filt'], params['pool_size'],
params['rnn_size'], params['fnn_size']
)
)
model = keras_model.get_model(data_in=data_in, data_out=data_out, dropout_rate=params['dropout_rate'],
nb_cnn2d_filt=params['nb_cnn2d_filt'], pool_size=params['pool_size'],
rnn_size=params['rnn_size'], fnn_size=params['fnn_size'],
classification_mode=params['mode'], weights=params['loss_weights'], summary=True)
if (os.path.exists('{}_model.ckpt'.format(unique_name))):
print("Model found!")
model.load_weights('{}_model.ckpt'.format(unique_name))
for i in range(10):
print("###")
best_metric = 99999
conf_mat = None
best_conf_mat = None
best_epoch = -1
patience_cnt = 0
epoch_metric_loss = np.zeros(params['nb_epochs'])
sed_score = np.zeros(params['nb_epochs'])
doa_score = np.zeros(params['nb_epochs'])
seld_score = np.zeros(params['nb_epochs'])
tr_loss = np.zeros(params['nb_epochs'])
val_loss = np.zeros(params['nb_epochs'])
doa_loss = np.zeros((params['nb_epochs'], 6))
sed_loss = np.zeros((params['nb_epochs'], 2))
for epoch_cnt in range(params['nb_epochs']):
start = time.time()
print("##### Training the model #####")
hist = model.fit_generator(
generator=data_gen_train.generate(),
steps_per_epoch=params['quick_test_steps'] if params[
'quick_test'] else data_gen_train.get_total_batches_in_data(),
validation_data=data_gen_test.generate(),
validation_steps=params['quick_test_steps'] if params[
'quick_test'] else data_gen_test.get_total_batches_in_data(),
use_multiprocessing=False,
workers=1,
epochs=1,
verbose=1
)
tr_loss[epoch_cnt] = hist.history.get('loss')[-1]
val_loss[epoch_cnt] = hist.history.get('val_loss')[-1]
print("##########################")
# Save, get model and re-load weights for the predict_generator bug
print("##### Saving weights #####")
model.save_weights('{}_model.ckpt'.format(unique_name))
model = keras_model.get_model(data_in=data_in, data_out=data_out, dropout_rate=params['dropout_rate'],
nb_cnn2d_filt=params['nb_cnn2d_filt'], pool_size=params['pool_size'],
rnn_size=params['rnn_size'], fnn_size=params['fnn_size'],
classification_mode=params['mode'], weights=params['loss_weights'], summary=False)
model.load_weights('{}_model.ckpt'.format(unique_name))
print("##########################")
print("#### Prediction on validation split ####")
pred = model.predict_generator(
generator=data_gen_test.generate(),
steps=params['quick_test_steps'] if params['quick_test'] else data_gen_test.get_total_batches_in_data(),
use_multiprocessing=False,
workers=1,
verbose=1
)
print("########################################")
# print("pred:",pred[1].shape)
if params['mode'] == 'regr':
sed_pred = np.array(evaluation_metrics.reshape_3Dto2D(pred[0])) > .5
doa_pred = evaluation_metrics.reshape_3Dto2D(pred[1])
# Old confidence intervals
'''
sed_err = sed_gt - sed_pred
[sed_conf_low, sed_conf_up, sed_median] = compute_confidence(sed_err)
# print("Condidence Interval for SED error is [" + str(sed_conf_low) + ", " + str(sed_conf_up) + "]")
print("Confidence Interval for SED error is [ %.5f, %.5f ]" % (sed_conf_low, sed_conf_up))
# print("\tMedian is " + str(sed_median))
print("\tMedian is %.5f" % (sed_median))
# print("\tDisplacement: +/- " + str(sed_conf_up - sed_median))
print("\tDisplacement: +/- %.5f" % (sed_conf_up - sed_median))
doa_err = doa_gt - doa_pred
[doa_conf_low, doa_conf_up, doa_median] = compute_confidence(doa_err)
# print("Condidence Interval for DOA is [" + str(doa_conf_low) + ", " + str(doa_conf_up) + "]")
print("Confidence Interval for DOA is [ %.5f, %.5f ]" % (doa_conf_low, doa_conf_up))
# print("Median is " + str(doa_median))
print("\tMedian is %.5f" % (doa_median))
# print("Displacement: +/- " + str(doa_conf_up - doa_median))
print("\tDisplacement: +/- %.5f" % (doa_conf_up - doa_median))
'''
sed_loss[epoch_cnt, :] = evaluation_metrics.compute_sed_scores(sed_pred, sed_gt,
data_gen_test.nb_frames_1s())
if params['azi_only']:
doa_loss[epoch_cnt, :], conf_mat = evaluation_metrics.compute_doa_scores_regr_xy(doa_pred, doa_gt,
sed_pred, sed_gt)
else:
doa_loss[epoch_cnt, :], conf_mat = evaluation_metrics.compute_doa_scores_regr_xyz(doa_pred, doa_gt,
sed_pred, sed_gt)
sed_score[epoch_cnt] = np.mean([sed_loss[epoch_cnt, 0], 1 - sed_loss[epoch_cnt, 1]])
doa_score[epoch_cnt] = np.mean([2 * np.arcsin(doa_loss[epoch_cnt, 1] / 2.0) / np.pi,
1 - (doa_loss[epoch_cnt, 5] / float(doa_gt.shape[0]))])
seld_score[epoch_cnt] = (sed_score[epoch_cnt] + doa_score[epoch_cnt]) / 2
if os.path.isdir('./models'):
plot.imshow(conf_mat, cmap='binary', interpolation='None')
plot.savefig('models/confusion_matrix.jpg')
# New confidence computation, differing doa and sed errors
sed_err = sed_loss[epoch_cnt, 0]
[sed_conf_low, sed_conf_up] = compute_confidence(sed_err, sed_pred.shape[0])
print("Confidence Interval for SED error is [ %f, %f ]" % (sed_conf_low, sed_conf_up))
#doa_err = doa_gt - doa_pred
#[x_err, y_err, z_err] = compute_doa_confidence(doa_err, n_classes)
plot_array = [tr_loss[epoch_cnt], # 0
val_loss[epoch_cnt], # 1
sed_loss[epoch_cnt][0], # 2 er
sed_loss[epoch_cnt][1], # 3 f1
doa_loss[epoch_cnt][0], # 4 avg_accuracy
doa_loss[epoch_cnt][1], # 5 doa_loss_gt
doa_loss[epoch_cnt][2], # 6 doa_loss_pred
doa_loss[epoch_cnt][3], # 7 doa_loss_gt_cnt
doa_loss[epoch_cnt][4], # 8 doa_loss_pred_cnt
doa_loss[epoch_cnt][5], # 9 good_frame_cnt
sed_score[epoch_cnt], # 10
doa_score[epoch_cnt],
seld_score[epoch_cnt],
#doa_conf_low, doa_median,
#doa_conf_up, sed_conf_low,
#sed_median, sed_conf_up]
sed_conf_low, sed_conf_up]
patience_cnt += 1
# model.save_weights('{}_model.ckpt'.format(unique_name))
simple_plotter.save_array_to_csv("{}_plot.csv".format(unique_name), plot_array)
#simple_plotter.plot_confidence(x_err, y_err, z_err, "ov")
print("##### Model and metrics saved! #####")
if seld_score[epoch_cnt] < best_metric:
best_metric = seld_score[epoch_cnt]
best_conf_mat = conf_mat
best_epoch = epoch_cnt
# Now we save the model at every iteration
model.save_weights('{}_BEST_model.ckpt'.format(unique_name))
patience_cnt = 0
print('epoch_cnt: %d, time: %.2fs, tr_loss: %.4f, val_loss: %.4f, '
'F1_overall: %.2f, ER_overall: %.2f, '
'doa_error_gt: %.2f, doa_error_pred: %.2f, good_pks_ratio:%.2f, '
'sed_score: %.4f, doa_score: %.4f, seld_score: %.4f, best_error_metric: %.2f, best_epoch : %d' %
(
epoch_cnt, time.time() - start, tr_loss[epoch_cnt], val_loss[epoch_cnt],
sed_loss[epoch_cnt, 1], sed_loss[epoch_cnt, 0],
doa_loss[epoch_cnt, 1], doa_loss[epoch_cnt, 2], doa_loss[epoch_cnt, 5] / float(sed_gt.shape[0]),
sed_score[epoch_cnt], doa_score[epoch_cnt], seld_score[epoch_cnt], best_metric, best_epoch
)
)
# plot_functions(unique_name, tr_loss, val_loss, sed_loss, doa_loss, sed_score, doa_score, epoch_cnt)
print('best_conf_mat : {}'.format(best_conf_mat))
print('best_conf_mat_diag : {}'.format(np.diag(best_conf_mat)))
print('saved model for the best_epoch: {} with best_metric: {}, '.format(best_epoch, best_metric))
print('DOA Metrics: doa_loss_gt: {}, doa_loss_pred: {}, good_pks_ratio: {}'.format(
doa_loss[best_epoch, 1], doa_loss[best_epoch, 2], doa_loss[best_epoch, 5] / float(sed_gt.shape[0])))
print('SED Metrics: ER_overall: {}, F1_overall: {}'.format(sed_loss[best_epoch, 0], sed_loss[best_epoch, 1]))
print('unique_name: {} '.format(unique_name))
def evaluate(model, data_gen_test, params, log_dir=".", unique_name="unique_name"):
logger.info("EVALUATE function called")
sed_gt, doa_gt = collect_ground_truth(data_gen_test, params)
predict_single_batch(model, data_gen_test)
dnn_output = model.predict(
x=data_gen_test.generate(),
steps=data_gen_test.get_total_batches_in_data(),
verbose=2
)
sed_pred = dnn_output[0] > 0.5
doa_pred = dnn_output[1]
sed_pred = evaluation_metrics.reshape_3Dto2D(sed_pred)
doa_pred = evaluation_metrics.reshape_3Dto2D(doa_pred)
num_classes = sed_pred.shape[-1]
num_dims_xyz = 3
if doa_pred.shape[-1] > num_classes * num_dims_xyz: # true means we are using masked mse
sed_mask = np.repeat(sed_pred, 3, -1)
doa_pred = doa_pred[..., num_classes:] * sed_mask
doa_gt = doa_gt[..., num_classes:] * sed_mask
sed_loss = evaluation_metrics.compute_sed_scores(sed_pred, sed_gt,
data_gen_test.nb_frames_1s())
if params['azi_only']:
doa_loss, conf_mat = evaluation_metrics.compute_doa_scores_regr_xy(doa_pred,
doa_gt,
sed_pred,
sed_gt)
else:
doa_loss, conf_mat = evaluation_metrics.compute_doa_scores_regr_xyz(doa_pred,
doa_gt,
sed_pred,
sed_gt)
epoch_metric_loss = np.mean([
sed_loss[0],
1 - sed_loss[1],
2 * np.arcsin(doa_loss[1] / 2.0) / np.pi,
1 - (doa_loss[5] / float(doa_gt.shape[0]))]
)
logger.info(
'F1_overall: %.2f, ER_overall: %.2f, '
'doa_error_gt: %.2f, doa_error_pred: %.2f, good_pks_ratio:%.2f, '
'error_metric: %.2f' %
(
sed_loss[1], sed_loss[0],
doa_loss[1], doa_loss[2], doa_loss[5] / float(sed_gt.shape[0]),
epoch_metric_loss
)
)
logger.info('DOA Metrics: doa_loss_gt: {}, doa_loss_pred: {}, good_pks_ratio: {}'.format(
doa_loss[1], doa_loss[2], doa_loss[5] / float(sed_gt.shape[0])))
logger.info(
'SED Metrics: F1_overall: {}, ER_overall: {}'.format(sed_loss[1], sed_loss[0]))
logger.info('unique_name: {} '.format(unique_name))
def train(model, data_gen_train, data_gen_val, params, log_dir=".", unique_name="unique_name"):
logger.info("Train function called")
sed_gt, doa_gt = collect_ground_truth(data_gen_val, params)
best_metric = 99999
conf_mat = None
best_conf_mat = None
best_epoch = -1
patience_cnt = 0
epoch_metric_loss = np.zeros(params['nb_epochs'])
tr_loss = np.zeros(params['nb_epochs'])
val_loss = np.zeros(params['nb_epochs'])
doa_loss = np.zeros((params['nb_epochs'], 6))
sed_loss = np.zeros((params['nb_epochs'], 2))
nb_epoch = params['nb_epochs']
model_path = os.path.join(log_dir, 'model')
K.clear_session()
for epoch_cnt in range(nb_epoch):
logger.info(f"Iteration {epoch_cnt}/{nb_epoch}")
start = time.time()
hist = model.fit(
x=data_gen_train.generate(),
steps_per_epoch=data_gen_train.get_total_batches_in_data(),
validation_data=data_gen_val.generate(),
validation_steps=data_gen_val.get_total_batches_in_data(),
epochs=params['epochs_per_iteration'],
verbose=2,
# callbacks=[MyCustomCallback]
)
tr_loss[epoch_cnt] = hist.history.get('loss')[-1]
val_loss[epoch_cnt] = hist.history.get('val_loss')[-1]
if (params['debug_load_single_batch']) and (epoch_cnt % 10 != 0) and (epoch_cnt != nb_epoch - 1):
plot_functions(os.path.join(log_dir, 'training_curves'), tr_loss, val_loss, sed_loss, doa_loss,
epoch_metric_loss)
else:
predict_single_batch(model, data_gen_train)
pred = model.predict(
x=data_gen_val.generate(),
steps=data_gen_val.get_total_batches_in_data(),
verbose=2
)
if params['mode'] == 'regr':
sed_pred = evaluation_metrics.reshape_3Dto2D(pred[0]) > 0.5
doa_pred = pred[1]
num_classes = sed_pred.shape[-1]
num_dims_xyz = 3
if doa_pred.shape[-1] > num_classes * num_dims_xyz: # true means we are using masked mse
doa_pred = doa_pred[..., num_classes:]
logger.debug(f"doa_pred.shape {doa_pred.shape}")
if doa_gt.shape[-1] > num_classes * num_dims_xyz: # true means we are using masked mse
doa_gt = doa_gt[..., num_classes:]
logger.debug(f"doa_gt.shape {doa_gt.shape}")
doa_pred = evaluation_metrics.reshape_3Dto2D(doa_pred)
sed_loss[epoch_cnt, :] = evaluation_metrics.compute_sed_scores(sed_pred, sed_gt,
data_gen_val.nb_frames_1s())
if params['azi_only']:
doa_loss[epoch_cnt, :], conf_mat = evaluation_metrics.compute_doa_scores_regr_xy(doa_pred,
doa_gt,
sed_pred,
sed_gt)
else:
doa_loss[epoch_cnt, :], conf_mat = evaluation_metrics.compute_doa_scores_regr_xyz(doa_pred,
doa_gt,
sed_pred,
sed_gt)
epoch_metric_loss[epoch_cnt] = np.mean([
sed_loss[epoch_cnt, 0],
1 - sed_loss[epoch_cnt, 1],
2 * np.arcsin(doa_loss[epoch_cnt, 1] / 2.0) / np.pi,
1 - (doa_loss[epoch_cnt, 5] / float(doa_gt.shape[0]))]
)
plot_functions(os.path.join(log_dir, 'training_curves'), tr_loss, val_loss, sed_loss, doa_loss,
epoch_metric_loss)
patience_cnt += 1
if (epoch_metric_loss[epoch_cnt] < best_metric):
best_metric = epoch_metric_loss[epoch_cnt]
best_conf_mat = conf_mat
best_epoch = epoch_cnt
model.save(model_path)
patience_cnt = 0
logger.info(
'epoch_cnt: %d, time: %.2fs, tr_loss: %.2f, val_loss: %.2f, '
'F1_overall: %.2f, ER_overall: %.2f, '
'doa_error_gt: %.2f, doa_error_pred: %.2f, good_pks_ratio:%.2f, '
'error_metric: %.2f, best_error_metric: %.2f, best_epoch : %d' %
(
epoch_cnt, time.time() - start, tr_loss[epoch_cnt], val_loss[epoch_cnt],
sed_loss[epoch_cnt, 1], sed_loss[epoch_cnt, 0],
doa_loss[epoch_cnt, 1], doa_loss[epoch_cnt, 2], doa_loss[epoch_cnt, 5] / float(sed_gt.shape[0]),
epoch_metric_loss[epoch_cnt], best_metric, best_epoch
)
)
if patience_cnt > params['patience']:
break
# otherwise RAM use increases after every epoch. But is the optimizer state forgotten?
K.clear_session()
if params['debug_load_single_batch'] and hist.history.get('loss')[-1] < 0.01:
break
if params['debug_load_single_batch']:
model.save(model_path)
else:
logger.info('best_conf_mat : {}'.format(best_conf_mat))
logger.info('best_conf_mat_diag : {}'.format(np.diag(best_conf_mat)))
logger.info('saved model for the best_epoch: {} with best_metric: {}, '.format(best_epoch, best_metric))
logger.info('DOA Metrics: doa_loss_gt: {}, doa_loss_pred: {}, good_pks_ratio: {}'.format(
doa_loss[best_epoch, 1], doa_loss[best_epoch, 2], doa_loss[best_epoch, 5] / float(sed_gt.shape[0])))
logger.info(
'SED Metrics: F1_overall: {}, ER_overall: {}'.format(sed_loss[best_epoch, 1], sed_loss[best_epoch, 0]))
np.save(os.path.join(log_dir, 'training-loss'), [tr_loss, val_loss])
logger.info(f'unique_name: {unique_name}')
logger.info(f'log_dir: {log_dir}')
def main(args):
"""
Main wrapper for training sound event localization and detection network.
:param argv: expects two optional inputs.
first input: job_id - (optional) all the output files will be uniquely represented with this. (default) 1
second input: task_id - (optional) To chose the system configuration in parameters.py.
(default) uses default parameters
"""
# use parameter set defined by user
task_id = args.params
params = parameter.get_params(task_id)
job_id = args.model_name
model_dir = 'models/' + args.author + '/' if args.author != "" else 'models/'
utils.create_folder(model_dir)
unique_name = '{}_ov{}_split{}_{}{}_3d{}_{}'.format(
params['dataset'], params['overlap'], params['split'], params['mode'],
params['weakness'], int(params['cnn_3d']), job_id)
model_name = unique_name
epoch_manager = JSON_Manager(args.author, unique_name)
logdir = "logs/" + args.author + "/" + unique_name
unique_name = os.path.join(model_dir, unique_name)
print("unique_name: {}\n".format(unique_name))
data_gen_test = cls_data_generator_seld.DataGenerator(
dataset=params['dataset'],
ov=params['overlap'],
split=params['split'],
db=params['db'],
nfft=params['nfft'],
batch_size=params['batch_size'],
seq_len=params['sequence_length'],
classifier_mode=params['mode'],
weakness=params['weakness'],
datagen_mode='test',
cnn3d=params['cnn_3d'],
xyz_def_zero=params['xyz_def_zero'],
azi_only=params['azi_only'],
shuffle=False)
data_in, data_out = data_gen_test.get_data_sizes()
print('FEATURES:\n'
'\tdata_in: {}\n'
'\tdata_out: {}\n'.format(data_in, data_out))
gt = collect_test_labels(data_gen_test, data_out, params['mode'],
params['quick_test'])
sed_gt = evaluation_metrics.reshape_3Dto2D(gt[0])
doa_gt = evaluation_metrics.reshape_3Dto2D(gt[1])
print('MODEL:\n'
'\tdropout_rate: {}\n'
'\tCNN: nb_cnn_filt: {}, pool_size{}\n'
'\trnn_size: {}, fnn_size: {}\n'.format(
params['dropout_rate'], params['nb_cnn3d_filt']
if params['cnn_3d'] else params['nb_cnn2d_filt'],
params['pool_size'], params['rnn_size'], params['fnn_size']))
model = load_model(os.path.join(model_dir, model_name + "_best_model.h5"),
custom_objects={
'QuaternionConv2D': QuaternionConv2D,
'QuaternionGRU': QuaternionGRU,
'QuaternionDense': QuaternionDense
})
model.summary()
plot_model(model, to_file=os.path.join(model_dir, 'model.png'))
best_metric = epoch_manager.get_best_metric()
conf_mat = None
best_conf_mat = epoch_manager.get_best_conf_mat()
best_epoch = epoch_manager.get_best_epoch()
patience_cnt = epoch_manager.get_patience_cnt()
epoch_metric_loss = np.zeros(params['nb_epochs'])
sed_score = np.zeros(params['nb_epochs'])
std_score = np.zeros(params['nb_epochs'])
doa_score = np.zeros(params['nb_epochs'])
seld_score = np.zeros(params['nb_epochs'])
tr_loss = np.zeros(params['nb_epochs'])
val_loss = np.zeros(params['nb_epochs'])
doa_loss = np.zeros((params['nb_epochs'], 6))
sed_loss = np.zeros((params['nb_epochs'], 2))
epoch_cnt = 0
pred = model.predict_generator(
generator=data_gen_test.generate(),
steps=data_gen_test.get_total_batches_in_data(),
use_multiprocessing=False,
verbose=2)
print("pred[1]:", pred[1].shape)
if params['mode'] == 'regr':
sed_pred = evaluation_metrics.reshape_3Dto2D(pred[0]) > 0.5
print(f"sed_pred: {sed_pred.shape}")
doa_pred = evaluation_metrics.reshape_3Dto2D(pred[1])
print(f"doa_pred: {doa_pred.shape}")
sed_loss[epoch_cnt, :] = evaluation_metrics.compute_sed_scores(
sed_pred, sed_gt, data_gen_test.nb_frames_1s())
if params['azi_only']:
doa_loss[
epoch_cnt, :], conf_mat = evaluation_metrics.compute_doa_scores_regr_xy(
doa_pred, doa_gt, sed_pred, sed_gt)
else:
doa_loss[
epoch_cnt, :], conf_mat = evaluation_metrics.compute_doa_scores_regr_xyz(
doa_pred, doa_gt, sed_pred, sed_gt)
sed_score[epoch_cnt] = np.mean(
[sed_loss[epoch_cnt, 0], 1 - sed_loss[epoch_cnt, 1]])
print(f"ER: {sed_loss[epoch_cnt, 0]}")
er = sed_loss[epoch_cnt, 0]
interval = 1.96 * np.sqrt(((er) * (1 - er)) / sed_pred.shape[0])
print(f"interval: {interval}")
doa_score[epoch_cnt] = np.mean([
2 * np.arcsin(doa_loss[epoch_cnt, 1] / 2.0) / np.pi,
1 - (doa_loss[epoch_cnt, 5] / float(doa_gt.shape[0]))
])
seld_score[epoch_cnt] = np.mean(
[sed_score[epoch_cnt], doa_score[epoch_cnt]])
doa_error = doa_pred - doa_gt
doa_error = np.reshape(doa_error, newshape=(doa_error.shape[0], 11, 2))
doa_error = np.absolute(doa_error[:, :, 0])
print(f"doa_error: {doa_error.shape}")
doa_error = np.reshape(doa_error,
newshape=(doa_error.shape[0] *
doa_error.shape[1]))
print(f"doa_error: {doa_error.shape}")
np.save(model_name + "_x", doa_error)
doa_error = doa_pred - doa_gt
doa_error = np.reshape(doa_error, newshape=(doa_error.shape[0], 11, 2))
doa_error = np.absolute(doa_error[:, :, 1])
print(f"doa_error: {doa_error.shape}")
doa_error = np.reshape(doa_error,
newshape=(doa_error.shape[0] *
doa_error.shape[1]))
print(f"doa_error: {doa_error.shape}")
np.save(model_name + "_y", doa_error)
# standard deviation
std_score[epoch_cnt] = np.std(
[sed_score[epoch_cnt], doa_score[epoch_cnt]])
print(f"{er-interval} / {er+interval}")
#lower_confidence, upper_confidence = evaluation_metrics.compute_confidence_interval(best_metric,best_std, params['nb_epochs'], confid_coeff=1.96) # 1.96 for a 95% CI
print("\n---- FINISHED ----\n")