def main(argv): """ Main wrapper for training sound event localization and detection network. :param argv: expects two optional inputs. first input: job_id - (optional) all the output files will be uniquely represented with this. (default) 1 second input: task_id - (optional) To chose the system configuration in parameters.py. (default) uses default parameters """ if len(argv) != 3: print('\n\n') print('-------------------------------------------------------------------------------------------------------') print('The code expected two inputs') print('\t>> python seld.py <job-id> <task-id>') print('\t\t<job-id> is a unique identifier which is used for output filenames (models, training plots). ' 'You can use any number or string for this.') print('\t\t<task-id> is used to choose the user-defined parameter set from parameter.py') print('Using default inputs for now') print('-------------------------------------------------------------------------------------------------------') print('\n\n') # use parameter set defined by user task_id = '1' if len(argv) < 3 else argv[-1] params = parameter.get_params(task_id) job_id = 1 if len(argv) < 2 else argv[1] model_dir = 'models/' utils.create_folder(model_dir) unique_name = '{}_ov{}_split{}_{}{}_3d{}_{}'.format( params['dataset'], params['overlap'], params['split'], params['mode'], params['weakness'], int(params['cnn_3d']), job_id ) unique_name = os.path.join(model_dir, unique_name) print("unique_name: {}\n".format(unique_name)) data_gen_train = cls_data_generator.DataGenerator( dataset=params['dataset'], ov=params['overlap'], split=params['split'], db=params['db'], nfft=params['nfft'], batch_size=params['batch_size'], seq_len=params['sequence_length'], classifier_mode=params['mode'], weakness=params['weakness'], datagen_mode='train', cnn3d=params['cnn_3d'], xyz_def_zero=params['xyz_def_zero'], azi_only=params['azi_only'] ) data_gen_test = cls_data_generator.DataGenerator( dataset=params['dataset'], ov=params['overlap'], split=params['split'], db=params['db'], nfft=params['nfft'], batch_size=params['batch_size'], seq_len=params['sequence_length'], classifier_mode=params['mode'], weakness=params['weakness'], datagen_mode='test', cnn3d=params['cnn_3d'], xyz_def_zero=params['xyz_def_zero'], azi_only=params['azi_only'], shuffle=False ) data_in, data_out = data_gen_train.get_data_sizes() print( 'FEATURES:\n' '\tdata_in: {}\n' '\tdata_out: {}\n'.format( data_in, data_out ) ) gt = collect_test_labels(data_gen_test, data_out, params['mode'], params['quick_test']) sed_gt = evaluation_metrics.reshape_3Dto2D(gt[0]) doa_gt = evaluation_metrics.reshape_3Dto2D(gt[1]) print( 'MODEL:\n' '\tdropout_rate: {}\n' '\tCNN: nb_cnn_filt: {}, pool_size{}\n' '\trnn_size: {}, fnn_size: {}\n'.format( params['dropout_rate'], params['nb_cnn3d_filt'] if params['cnn_3d'] else params['nb_cnn2d_filt'], params['pool_size'], params['rnn_size'], params['fnn_size'] ) ) model = keras_model.get_model(data_in=data_in, data_out=data_out, dropout_rate=params['dropout_rate'], nb_cnn2d_filt=params['nb_cnn2d_filt'], pool_size=params['pool_size'], rnn_size=params['rnn_size'], fnn_size=params['fnn_size'], classification_mode=params['mode'], weights=params['loss_weights'], loader=False, loader2=False) # Change loader to True to enable transfer learning, Change loader2 to True to enable transfer learning with different labels best_metric = 99999 conf_mat = None best_conf_mat = None best_epoch = -1 patience_cnt = 0 epoch_metric_loss = np.zeros(params['nb_epochs']) tr_loss = np.zeros(params['nb_epochs']) val_loss = np.zeros(params['nb_epochs']) doa_loss = np.zeros((params['nb_epochs'], 6)) sed_loss = np.zeros((params['nb_epochs'], 2)) nb_epoch = 2 if params['quick_test'] else params['nb_epochs'] for epoch_cnt in range(nb_epoch): start = time.time() hist = model.fit_generator( generator=data_gen_train.generate(), steps_per_epoch=2 if params['quick_test'] else data_gen_train.get_total_batches_in_data(), validation_data=data_gen_test.generate(), validation_steps=2 if params['quick_test'] else data_gen_test.get_total_batches_in_data(), epochs=1, verbose=0 ) tr_loss[epoch_cnt] = hist.history.get('loss')[-1] val_loss[epoch_cnt] = hist.history.get('val_loss')[-1] pred = model.predict_generator( generator=data_gen_test.generate(), steps=2 if params['quick_test'] else data_gen_test.get_total_batches_in_data(), verbose=2 ) if params['mode'] == 'regr': sed_pred = evaluation_metrics.reshape_3Dto2D(pred[0]) > 0.5 doa_pred = evaluation_metrics.reshape_3Dto2D(pred[1]) sed_loss[epoch_cnt, :] = evaluation_metrics.compute_sed_scores(sed_pred, sed_gt, data_gen_test.nb_frames_1s()) if params['azi_only']: doa_loss[epoch_cnt, :], conf_mat = evaluation_metrics.compute_doa_scores_regr_xy(doa_pred, doa_gt, sed_pred, sed_gt) else: doa_loss[epoch_cnt, :], conf_mat = evaluation_metrics.compute_doa_scores_regr_xyz(doa_pred, doa_gt, sed_pred, sed_gt) epoch_metric_loss[epoch_cnt] = np.mean([ sed_loss[epoch_cnt, 0], 1-sed_loss[epoch_cnt, 1], 2*np.arcsin(doa_loss[epoch_cnt, 1]/2.0)/np.pi, 1 - (doa_loss[epoch_cnt, 5] / float(doa_gt.shape[0]))] ) plot_functions(unique_name, tr_loss, val_loss, sed_loss, doa_loss, epoch_metric_loss) patience_cnt += 1 if epoch_metric_loss[epoch_cnt] < best_metric: best_metric = epoch_metric_loss[epoch_cnt] best_conf_mat = conf_mat best_epoch = epoch_cnt model.save('{}_model.h5'.format(unique_name)) patience_cnt = 0 print( 'epoch_cnt: %d, time: %.2fs, tr_loss: %.2f, val_loss: %.2f, ' 'F1_overall: %.2f, ER_overall: %.2f, ' 'doa_error_gt: %.2f, doa_error_pred: %.2f, good_pks_ratio:%.2f, ' 'error_metric: %.2f, best_error_metric: %.2f, best_epoch : %d' % ( epoch_cnt, time.time() - start, tr_loss[epoch_cnt], val_loss[epoch_cnt], sed_loss[epoch_cnt, 1], sed_loss[epoch_cnt, 0], doa_loss[epoch_cnt, 1], doa_loss[epoch_cnt, 2], doa_loss[epoch_cnt, 5] / float(sed_gt.shape[0]), epoch_metric_loss[epoch_cnt], best_metric, best_epoch ) ) if patience_cnt > params['patience']: break print('best_conf_mat : {}'.format(best_conf_mat)) print('best_conf_mat_diag : {}'.format(np.diag(best_conf_mat))) print('saved model for the best_epoch: {} with best_metric: {}, '.format(best_epoch, best_metric)) print('DOA Metrics: doa_loss_gt: {}, doa_loss_pred: {}, good_pks_ratio: {}'.format( doa_loss[best_epoch, 1], doa_loss[best_epoch, 2], doa_loss[best_epoch, 5] / float(sed_gt.shape[0]))) print('SED Metrics: ER_overall: {}, F1_overall: {}'.format(sed_loss[best_epoch, 1], sed_loss[best_epoch, 0])) print('unique_name: {} '.format(unique_name))
def main(argv): """ Main wrapper for training sound event localization and detection network. :param argv: expects two optional inputs. first input: job_id - (optional) all the output files will be uniquely represented with this. (default) 1 second input: task_id - (optional) To chose the system configuration in parameters.py. (default) uses default parameters """ if len(argv) != 3: print('\n\n') print( '-------------------------------------------------------------------------------------------------------' ) print('The code expected two inputs') print('\t>> python seld.py <job-id> <task-id>') print( '\t\t<job-id> is a unique identifier which is used for output filenames (models, training plots). ' 'You can use any number or string for this.') print( '\t\t<task-id> is used to choose the user-defined parameter set from parameter.py' ) print('Using default inputs for now') print( '-------------------------------------------------------------------------------------------------------' ) print('\n\n') # use parameter set defined by user task_id = '1' if len(argv) < 3 else argv[-1] params = parameter.get_params(task_id) job_id = 1 if len(argv) < 2 else argv[1] model_dir = 'models/' utils.create_folder(model_dir) unique_name = '{}_ov{}_split{}_{}{}_3d{}_{}'.format( params['dataset'], params['overlap'], params['split'], params['mode'], params['weakness'], int(params['cnn_3d']), job_id) unique_name = os.path.join(model_dir, unique_name) print("unique_name: {}\n".format(unique_name)) data_gen_train = cls_data_generator.DataGenerator( dataset=params['dataset'], ov=params['overlap'], split=params['split'], db=params['db'], nfft=params['nfft'], batch_size=params['batch_size'], seq_len=params['sequence_length'], classifier_mode=params['mode'], weakness=params['weakness'], datagen_mode='train', cnn3d=params['cnn_3d'], xyz_def_zero=params['xyz_def_zero'], azi_only=params['azi_only']) data_gen_test = cls_data_generator.DataGenerator( dataset=params['dataset'], ov=params['overlap'], split=params['split'], db=params['db'], nfft=params['nfft'], batch_size=params['batch_size'], seq_len=params['sequence_length'], classifier_mode=params['mode'], weakness=params['weakness'], datagen_mode='test', cnn3d=params['cnn_3d'], xyz_def_zero=params['xyz_def_zero'], azi_only=params['azi_only'], shuffle=False) data_in, data_out = data_gen_train.get_data_sizes() print('FEATURES:\n' '\tdata_in: {}\n' '\tdata_out: {}\n'.format(data_in, data_out)) gt = collect_test_labels(data_gen_test, data_out, params['mode'], params['quick_test']) sed_gt = evaluation_metrics.reshape_3Dto2D(gt[0]) doa_gt = evaluation_metrics.reshape_3Dto2D(gt[1]) print('MODEL:\n' '\tdropout_rate: {}\n' '\tCNN: nb_cnn_filt: {}, pool_size{}\n' '\trnn_size: {}, fnn_size: {}\n'.format( params['dropout_rate'], params['nb_cnn3d_filt'] if params['cnn_3d'] else params['nb_cnn2d_filt'], params['pool_size'], params['rnn_size'], params['fnn_size'])) # TPU CODE FOR GOOGLE COLABORATORY resolver = tf.distribute.cluster_resolver.TPUClusterResolver( tpu='grpc://' + os.environ['COLAB_TPU_ADDR']) tf.config.experimental_connect_to_cluster(resolver) # This is the TPU initialization code that has to be at the beginning. tf.tpu.experimental.initialize_tpu_system(resolver) print("All devices: ", tf.config.list_logical_devices('TPU')) strategy = tf.distribute.experimental.TPUStrategy(resolver) with strategy.scope(): # Load or create model model = utils.load_model(unique_name) if model is None: model = keras_model.get_model( data_in=data_in, data_out=data_out, dropout_rate=params['dropout_rate'], nb_cnn2d_filt=params['nb_cnn2d_filt'], pool_size=params['pool_size'], rnn_size=params['rnn_size'], fnn_size=params['fnn_size'], classification_mode=params['mode'], weights=params['loss_weights']) model.summary() best_metric = 99999 conf_mat = None best_conf_mat = None best_epoch = -1 patience_cnt = 0 epoch_metric_loss = np.zeros(params['nb_epochs']) sed_score = np.zeros(params['nb_epochs']) doa_score = np.zeros(params['nb_epochs']) tr_loss = np.zeros(params['nb_epochs']) val_loss = np.zeros(params['nb_epochs']) doa_loss = np.zeros((params['nb_epochs'], 6)) sed_loss = np.zeros((params['nb_epochs'], 2)) for epoch_cnt in range(params['nb_epochs']): start = time.time() hist = model.fit_generator( generator=data_gen_train.generate(), steps_per_epoch=5 if params['quick_test'] else data_gen_train.get_total_batches_in_data(), validation_data=data_gen_test.generate(), validation_steps=5 if params['quick_test'] else data_gen_test.get_total_batches_in_data(), use_multiprocessing=False, epochs=1, verbose=1) tr_loss[epoch_cnt] = hist.history.get('loss')[-1] val_loss[epoch_cnt] = hist.history.get('val_loss')[-1] pred = model.predict_generator( generator=data_gen_test.generate(), steps=5 if params['quick_test'] else data_gen_test.get_total_batches_in_data(), use_multiprocessing=False, verbose=2) print("pred:", pred[1].shape) if params['mode'] == 'regr': sed_pred = evaluation_metrics.reshape_3Dto2D(pred[0]) > 0.5 doa_pred = evaluation_metrics.reshape_3Dto2D(pred[1]) sed_loss[epoch_cnt, :] = evaluation_metrics.compute_sed_scores( sed_pred, sed_gt, data_gen_test.nb_frames_1s()) if params['azi_only']: doa_loss[ epoch_cnt, :], conf_mat = evaluation_metrics.compute_doa_scores_regr_xy( doa_pred, doa_gt, sed_pred, sed_gt) else: doa_loss[ epoch_cnt, :], conf_mat = evaluation_metrics.compute_doa_scores_regr_xyz( doa_pred, doa_gt, sed_pred, sed_gt) # epoch_metric_loss[epoch_cnt] = np.mean([ # sed_loss[epoch_cnt, 0], # 1-sed_loss[epoch_cnt, 1], # 2*np.arcsin(doa_loss[epoch_cnt, 1]/2.0)/np.pi, # 1 - (doa_loss[epoch_cnt, 5] / float(doa_gt.shape[0]))] # ) sed_score[epoch_cnt] = np.mean( [sed_loss[epoch_cnt, 0], 1 - sed_loss[epoch_cnt, 1]]) doa_score[epoch_cnt] = np.mean([ 2 * np.arcsin(doa_loss[epoch_cnt, 1] / 2.0) / np.pi, 1 - (doa_loss[epoch_cnt, 5] / float(doa_gt.shape[0])) ]) #plot_functions(unique_name, tr_loss, val_loss, sed_loss, doa_loss, epoch_metric_loss) plot_functions(unique_name, tr_loss, val_loss, sed_loss, doa_loss, sed_score, doa_score) patience_cnt += 1 # if epoch_metric_loss[epoch_cnt] < best_metric: # best_metric = epoch_metric_loss[epoch_cnt] # best_conf_mat = conf_mat # best_epoch = epoch_cnt # model.save('{}_model.h5'.format(unique_name)) # patience_cnt = 0 if sed_score[epoch_cnt] < best_metric: best_metric = sed_score[epoch_cnt] best_conf_mat = conf_mat best_epoch = epoch_cnt model.save('{}_model.h5'.format(unique_name)) patience_cnt = 0 print( 'epoch_cnt: %d, time: %.2fs, tr_loss: %.2f, val_loss: %.2f, ' 'F1_overall: %.2f, ER_overall: %.2f, ' 'doa_error_gt: %.2f, doa_error_pred: %.2f, good_pks_ratio:%.2f, ' 'sed_score: %.2f, doa_score: %.2f, best_error_metric: %.2f, best_epoch : %d' % (epoch_cnt, time.time() - start, tr_loss[epoch_cnt], val_loss[epoch_cnt], sed_loss[epoch_cnt, 1], sed_loss[epoch_cnt, 0], doa_loss[epoch_cnt, 1], doa_loss[epoch_cnt, 2], doa_loss[epoch_cnt, 5] / float(sed_gt.shape[0]), sed_score[epoch_cnt], doa_score[epoch_cnt], best_metric, best_epoch)) #plot_functions(unique_name, tr_loss, val_loss, sed_loss, doa_loss, sed_score, doa_score, epoch_cnt) print('best_conf_mat : {}'.format(best_conf_mat)) print('best_conf_mat_diag : {}'.format(np.diag(best_conf_mat))) print('saved model for the best_epoch: {} with best_metric: {}, '.format( best_epoch, best_metric)) print( 'DOA Metrics: doa_loss_gt: {}, doa_loss_pred: {}, good_pks_ratio: {}'. format(doa_loss[best_epoch, 1], doa_loss[best_epoch, 2], doa_loss[best_epoch, 5] / float(sed_gt.shape[0]))) print('SED Metrics: ER_overall: {}, F1_overall: {}'.format( sed_loss[best_epoch, 0], sed_loss[best_epoch, 1])) print('unique_name: {} '.format(unique_name))
def main(args): """ Main wrapper for training sound event localization and detection network. :param argv: expects two optional inputs. first input: job_id - (optional) all the output files will be uniquely represented with this. (default) 1 second input: task_id - (optional) To chose the system configuration in parameters.py. (default) uses default parameters """ # use parameter set defined by user task_id = args.params params = parameter.get_params(task_id) job_id = args.model_name model_dir = 'models/' + args.author + '/' if args.author != "" else 'models/' utils.create_folder(model_dir) unique_name = '{}_ov{}_split{}_{}{}_3d{}_{}'.format( params['dataset'], params['overlap'], params['split'], params['mode'], params['weakness'], int(params['cnn_3d']), job_id) model_name = unique_name epoch_manager = JSON_Manager(args.author, unique_name) logdir = "logs/" + args.author + "/" + unique_name unique_name = os.path.join(model_dir, unique_name) print("unique_name: {}\n".format(unique_name)) session = tf.InteractiveSession() file_writer = tf.summary.FileWriter(logdir, session.graph) data_gen_train = cls_data_generator.DataGenerator( dataset=params['dataset'], ov=params['overlap'], split=params['split'], db=params['db'], nfft=params['nfft'], batch_size=params['batch_size'], seq_len=params['sequence_length'], classifier_mode=params['mode'], weakness=params['weakness'], datagen_mode='train', cnn3d=params['cnn_3d'], xyz_def_zero=params['xyz_def_zero'], azi_only=params['azi_only']) data_gen_test = cls_data_generator.DataGenerator( dataset=params['dataset'], ov=params['overlap'], split=params['split'], db=params['db'], nfft=params['nfft'], batch_size=params['batch_size'], seq_len=params['sequence_length'], classifier_mode=params['mode'], weakness=params['weakness'], datagen_mode='test', cnn3d=params['cnn_3d'], xyz_def_zero=params['xyz_def_zero'], azi_only=params['azi_only'], shuffle=False) data_in, data_out = data_gen_train.get_data_sizes() print('FEATURES:\n' '\tdata_in: {}\n' '\tdata_out: {}\n'.format(data_in, data_out)) gt = collect_test_labels(data_gen_test, data_out, params['mode'], params['quick_test']) sed_gt = evaluation_metrics.reshape_3Dto2D(gt[0]) doa_gt = evaluation_metrics.reshape_3Dto2D(gt[1]) print('MODEL:\n' '\tdropout_rate: {}\n' '\tCNN: nb_cnn_filt: {}, pool_size{}\n' '\trnn_size: {}, fnn_size: {}\n'.format( params['dropout_rate'], params['nb_cnn3d_filt'] if params['cnn_3d'] else params['nb_cnn2d_filt'], params['pool_size'], params['rnn_size'], params['fnn_size'])) model = keras_model.get_model(data_in=data_in, data_out=data_out, dropout_rate=params['dropout_rate'], nb_cnn2d_filt=params['nb_cnn2d_filt'], pool_size=params['pool_size'], rnn_size=params['rnn_size'], fnn_size=params['fnn_size'], classification_mode=params['mode'], weights=params['loss_weights']) initial = epoch_manager.get_epoch() if initial != 0: print(f"Resume training from epoch {initial}") print("Loading already trained model...") # In order to load custom layers we need to link the references to the custom objects model = load_model(os.path.join(model_dir, model_name + "_model.h5"), custom_objects={ 'QuaternionConv2D': QuaternionConv2D, 'QuaternionGRU': QuaternionGRU, 'QuaternionDense': QuaternionDense }) best_metric = epoch_manager.get_best_metric() best_std = epoch_manager.get_best_std() conf_mat = None best_conf_mat = epoch_manager.get_best_conf_mat() best_epoch = epoch_manager.get_best_epoch() patience_cnt = epoch_manager.get_patience_cnt() epoch_metric_loss = np.zeros(params['nb_epochs']) sed_score = np.zeros(params['nb_epochs']) std_score = np.zeros(params['nb_epochs']) doa_score = np.zeros(params['nb_epochs']) seld_score = np.zeros(params['nb_epochs']) tr_loss = np.zeros(params['nb_epochs']) val_loss = np.zeros(params['nb_epochs']) doa_loss = np.zeros((params['nb_epochs'], 6)) sed_loss = np.zeros((params['nb_epochs'], 2)) time_hold = tf.placeholder(tf.float32, shape=None, name='time_summary') time_summ = tf.summary.scalar('time', time_hold) tr_loss_hold = tf.placeholder(tf.float32, shape=None, name='tr_loss_summary') tr_loss_summ = tf.summary.scalar('tr_loss', tr_loss_hold) val_loss_hold = tf.placeholder(tf.float32, shape=None, name='val_loss_summary') val_loss_summ = tf.summary.scalar('val_loss', val_loss_hold) f1_hold = tf.placeholder(tf.float32, shape=None, name='f1_summary') f1_summ = tf.summary.scalar('F1_overall', f1_hold) er_hold = tf.placeholder(tf.float32, shape=None, name='er_summary') er_summ = tf.summary.scalar('ER_overall', er_hold) doa_error_gt_hold = tf.placeholder(tf.float32, shape=None, name='doa_error_gt_summary') doa_error_gt_summ = tf.summary.scalar('doa_error_gt', doa_error_gt_hold) doa_error_pred_hold = tf.placeholder(tf.float32, shape=None, name='doa_error_pred_summary') doa_error_pred_summ = tf.summary.scalar('doa_error_pred', doa_error_pred_hold) good_pks_hold = tf.placeholder(tf.float32, shape=None, name='good_pks_summary') good_pks_summ = tf.summary.scalar('good_pks_ratio', good_pks_hold) sed_score_hold = tf.placeholder(tf.float32, shape=None, name='sed_score_summary') sed_score_summ = tf.summary.scalar('sed_score', sed_score_hold) doa_score_hold = tf.placeholder(tf.float32, shape=None, name='doa_score_summary') doa_score_summ = tf.summary.scalar('doa_score', doa_score_hold) seld_score_hold = tf.placeholder(tf.float32, shape=None, name='seld_score_summary') seld_score_summ = tf.summary.scalar('seld_score', seld_score_hold) std_score_hold = tf.placeholder(tf.float32, shape=None, name='std_score_summary') std_score_summ = tf.summary.scalar('std_score', std_score_hold) best_error_metric_hold = tf.placeholder(tf.float32, shape=None, name='best_error_metric_summary') best_error_metric_summ = tf.summary.scalar('best_error_metric', best_error_metric_hold) best_epoch_hold = tf.placeholder(tf.float32, shape=None, name='best_epoch_summary') best_epoch_summ = tf.summary.scalar('best_epoch', best_epoch_hold) best_std_hold = tf.placeholder(tf.float32, shape=None, name='best_std_summary') best_std_summ = tf.summary.scalar('best_std', best_std_hold) merged = tf.summary.merge_all() for epoch_cnt in range(initial, params['nb_epochs']): start = time.time() hist = model.fit_generator( generator=data_gen_train.generate(), steps_per_epoch=5 if params['quick_test'] else data_gen_train.get_total_batches_in_data(), validation_data=data_gen_test.generate(), validation_steps=5 if params['quick_test'] else data_gen_test.get_total_batches_in_data(), use_multiprocessing=False, epochs=1, verbose=1) tr_loss[epoch_cnt] = hist.history.get('loss')[-1] val_loss[epoch_cnt] = hist.history.get('val_loss')[-1] pred = model.predict_generator( generator=data_gen_test.generate(), steps=5 if params['quick_test'] else data_gen_test.get_total_batches_in_data(), use_multiprocessing=False, verbose=2) print("pred:", pred[1].shape) if params['mode'] == 'regr': sed_pred = evaluation_metrics.reshape_3Dto2D(pred[0]) > 0.5 doa_pred = evaluation_metrics.reshape_3Dto2D(pred[1]) sed_loss[epoch_cnt, :] = evaluation_metrics.compute_sed_scores( sed_pred, sed_gt, data_gen_test.nb_frames_1s()) if params['azi_only']: doa_loss[ epoch_cnt, :], conf_mat = evaluation_metrics.compute_doa_scores_regr_xy( doa_pred, doa_gt, sed_pred, sed_gt) else: doa_loss[ epoch_cnt, :], conf_mat = evaluation_metrics.compute_doa_scores_regr_xyz( doa_pred, doa_gt, sed_pred, sed_gt) sed_score[epoch_cnt] = np.mean( [sed_loss[epoch_cnt, 0], 1 - sed_loss[epoch_cnt, 1]]) doa_score[epoch_cnt] = np.mean([ 2 * np.arcsin(doa_loss[epoch_cnt, 1] / 2.0) / np.pi, 1 - (doa_loss[epoch_cnt, 5] / float(doa_gt.shape[0])) ]) seld_score[epoch_cnt] = np.mean( [sed_score[epoch_cnt], doa_score[epoch_cnt]]) # standard deviation std_score[epoch_cnt] = np.std( [sed_score[epoch_cnt], doa_score[epoch_cnt]]) plot_functions(unique_name, tr_loss, val_loss, sed_loss, doa_loss, sed_score, doa_score) patience_cnt += 1 epoch_manager.increase_patience_cnt() model.save('{}_model.h5'.format(unique_name)) if seld_score[epoch_cnt] < best_metric: best_metric = seld_score[epoch_cnt] epoch_manager.set_best_metric(best_metric) best_std = std_score[epoch_cnt] epoch_manager.set_best_std(best_std) best_conf_mat = conf_mat epoch_manager.set_best_conf_mat(conf_mat) best_epoch = epoch_cnt epoch_manager.set_best_epoch(best_epoch) model.save('{}_best_model.h5'.format(unique_name)) patience_cnt = 0 epoch_manager.reset_patience_cnt() if patience_cnt > params['patience']: print( f"\n---- PATIENCE TRIGGERED AFTER {epoch_cnt} EPOCHS ----\n") break summary = session.run(merged, feed_dict={ time_hold: time.time() - start, tr_loss_hold: tr_loss[epoch_cnt], val_loss_hold: val_loss[epoch_cnt], f1_hold: sed_loss[epoch_cnt, 1], er_hold: sed_loss[epoch_cnt, 0], doa_error_gt_hold: doa_loss[epoch_cnt, 1], doa_error_pred_hold: doa_loss[epoch_cnt, 2], good_pks_hold: doa_loss[epoch_cnt, 5] / float(sed_gt.shape[0]), sed_score_hold: sed_score[epoch_cnt], doa_score_hold: doa_score[epoch_cnt], seld_score_hold: seld_score[epoch_cnt], std_score_hold: std_score[epoch_cnt], best_error_metric_hold: best_metric, best_epoch_hold: best_epoch, best_std_hold: best_std }) file_writer.add_summary(summary, epoch_cnt) print( 'epoch_cnt: %d, time: %.2fs, tr_loss: %.2f, val_loss: %.2f, ' 'F1_overall: %.2f, ER_overall: %.2f, ' 'doa_error_gt: %.2f, doa_error_pred: %.2f, good_pks_ratio:%.2f, ' 'sed_score: %.2f, doa_score: %.2f, best_error_metric: %.2f, best_epoch : %d, best_std: %.2f' % (epoch_cnt, time.time() - start, tr_loss[epoch_cnt], val_loss[epoch_cnt], sed_loss[epoch_cnt, 1], sed_loss[epoch_cnt, 0], doa_loss[epoch_cnt, 1], doa_loss[epoch_cnt, 2], doa_loss[epoch_cnt, 5] / float(sed_gt.shape[0]), sed_score[epoch_cnt], doa_score[epoch_cnt], best_metric, best_epoch, best_std)) epoch_manager.increase_epoch() lower_confidence, upper_confidence = evaluation_metrics.compute_confidence_interval( best_metric, best_std, params['nb_epochs'], confid_coeff=1.96) # 1.96 for a 95% CI print("\n---- FINISHED TRAINING ----\n") print('best_conf_mat : {}'.format(best_conf_mat)) print('best_conf_mat_diag : {}'.format(np.diag(best_conf_mat))) print('saved model for the best_epoch: {} with best_metric: {}, '.format( best_epoch, best_metric)) print( 'DOA Metrics: doa_loss_gt: {}, doa_loss_pred: {}, good_pks_ratio: {}'. format(doa_loss[best_epoch, 1], doa_loss[best_epoch, 2], doa_loss[best_epoch, 5] / float(sed_gt.shape[0]))) print('SED Metrics: ER_overall: {}, F1_overall: {}'.format( sed_loss[best_epoch, 0], sed_loss[best_epoch, 1])) print('Confidence Interval: lower_interval: {}, upper_inteval: {}'.format( lower_confidence, upper_confidence)) print('unique_name: {} '.format(unique_name))
def main(argv): """ Main wrapper for training sound event localization and detection network. :param argv: expects two optional inputs. first input: job_id - (optional) all the output files will be uniquely represented with this. (default) 1 second input: task_id - (optional) To chose the system configuration in parameters.py. (default) uses default parameters """ if len(argv) != 3: print('\n\n') print('-------------------------------------------------------------------------------------------------------') print('The code expected two inputs') print('\t>> python seld.py <job-id> <task-id>') print('\t\t<job-id> is a unique identifier which is used for output filenames (models, training plots). ' 'You can use any number or string for this.') print('\t\t<task-id> is used to choose the user-defined parameter set from parameter.py') print('Using default inputs for now') print('-------------------------------------------------------------------------------------------------------') print('\n\n') # use parameter set defined by user task_id = '1' if len(argv) < 3 else argv[-1] params = parameter.get_params(task_id) job_id = 1 if len(argv) < 2 else argv[1] model_dir = 'models/' utils.create_folder(model_dir) unique_name = '{}_train{}_validation{}_seq{}'.format(params['dataset'], params['train_split'], params['val_split'], params['sequence_length']) unique_name = os.path.join(model_dir, unique_name) print("unique_name: {}\n".format(unique_name)) # Cycling over overlaps for ov in range(1, params['overlap']+1): data_gen_test = cls_data_generator.DataGenerator( dataset=params['dataset'], ov=params['overlap'], ov_num=ov, split=params['test_split'], db=params['db'], nfft=params['nfft'], batch_size=params['batch_size'], seq_len=params['sequence_length'], classifier_mode=params['mode'], weakness=params['weakness'], datagen_mode='test', cnn3d=params['cnn_3d'], xyz_def_zero=params['xyz_def_zero'], azi_only=params['azi_only'], shuffle=False ) data_in, data_out = data_gen_test.get_data_sizes() n_classes = data_out[0][2] print( 'FEATURES:\n' '\tdata_in: {}\n' '\tdata_out: {}\n'.format( data_in, data_out ) ) gt = collect_test_labels(data_gen_test, data_out, params['mode'], params['quick_test']) sed_gt = evaluation_metrics.reshape_3Dto2D(gt[0]) doa_gt = evaluation_metrics.reshape_3Dto2D(gt[1]) print("#### Saving DOA and SED GT Values ####") f = open("models/doa_gt.txt", "w+") for elem in doa_gt: f.write(str(list(elem)) + "\n") f.close() f = open("models/sed_gt.txt", "w+") for elem in sed_gt: f.write(str(elem)+"\n") f.close() print("######################################") print( 'MODEL:\n' '\tdropout_rate: {}\n' '\tCNN: nb_cnn_filt: {}, pool_size{}\n' '\trnn_size: {}, fnn_size: {}\n'.format( params['dropout_rate'], params['nb_cnn3d_filt'] if params['cnn_3d'] else params['nb_cnn2d_filt'], params['pool_size'], params['rnn_size'], params['fnn_size'] ) ) model = keras_model.get_model(data_in=data_in, data_out=data_out, dropout_rate=params['dropout_rate'], nb_cnn2d_filt=params['nb_cnn2d_filt'], pool_size=params['pool_size'], rnn_size=params['rnn_size'], fnn_size=params['fnn_size'], classification_mode=params['mode'], weights=params['loss_weights'], summary=False) if(os.path.exists('{}_model.ckpt'.format(unique_name))): print("Model found!") model.load_weights('{}_model.ckpt'.format(unique_name)) for i in range(10): print("###") sed_score = np.zeros(params['nb_epochs']) doa_score = np.zeros(params['nb_epochs']) seld_score = np.zeros(params['nb_epochs']) tr_loss = np.zeros(params['nb_epochs']) val_loss = np.zeros(params['nb_epochs']) doa_loss = np.zeros((params['nb_epochs'], 6)) sed_loss = np.zeros((params['nb_epochs'], 2)) epoch_cnt = 0 start = time.time() print("#### Prediction on validation split ####") pred = model.predict_generator( generator=data_gen_test.generate(), steps=params['quick_test_steps'] if params['quick_test'] else data_gen_test.get_total_batches_in_data(), use_multiprocessing=False, workers=1, verbose=1, ) print("##########################") #print("pred:", pred[1].shape) if params['mode'] == 'regr': sed_pred = np.array(evaluation_metrics.reshape_3Dto2D(pred[0])) > .5 doa_pred = evaluation_metrics.reshape_3Dto2D(pred[1]) print("#### Saving DOA and SED Pred Values ####") f = open("models/doa_pred.txt", "w+") for elem in doa_pred: f.write(str(list(elem)) + "\n") f.close() f = open("models/sed_pred.txt", "w+") for elem in sed_pred: f.write(str(elem)+"\n") f.close() print("########################################") # Old version of confidence intervals ''' # Computing confidence intervals sed_err = sed_gt - sed_pred [sed_conf_low, sed_conf_up, sed_median] = compute_confidence(sed_err) # print("Condidence Interval for SED error is [" + str(sed_conf_low) + ", " + str(sed_conf_up) + "]") print("Confidence Interval for SED error is [ %f, %f ]" % (sed_conf_low, sed_conf_up)) # print("\tMedian is " + str(sed_median)) print("\tMedian is %f" % (sed_median)) # print("\tDisplacement: +/- " + str(sed_conf_up - sed_median)) print("\tDisplacement: +/- %f" % (sed_conf_up - sed_median)) doa_err = doa_gt - doa_pred [doa_conf_low, doa_conf_up, doa_median] = compute_confidence(doa_err) # print("Condidence Interval for DOA is [" + str(doa_conf_low) + ", " + str(doa_conf_up) + "]") print("Confidence Interval for DOA is [ %f, %f ]" % (doa_conf_low, doa_conf_up)) # print("Median is " + str(doa_median)) print("\tMedian is %f" % (doa_median)) # print("Displacement: +/- " + str(doa_conf_up - doa_median)) print("\tDisplacement: +/- %f" % (doa_conf_up - doa_median)) # ------------------------------ ''' sed_loss[epoch_cnt, :] = evaluation_metrics.compute_sed_scores(sed_pred, sed_gt, data_gen_test.nb_frames_1s()) if params['azi_only']: doa_loss[epoch_cnt, :], conf_mat = evaluation_metrics.compute_doa_scores_regr_xy(doa_pred, doa_gt, sed_pred, sed_gt) else: doa_loss[epoch_cnt, :], conf_mat = evaluation_metrics.compute_doa_scores_regr_xyz(doa_pred, doa_gt, sed_pred, sed_gt) sed_score[epoch_cnt] = np.mean([sed_loss[epoch_cnt, 0], 1 - sed_loss[epoch_cnt, 1]]) doa_score[epoch_cnt] = np.mean([2 * np.arcsin(doa_loss[epoch_cnt, 1] / 2.0) / np.pi, 1 - (doa_loss[epoch_cnt, 5] / float(doa_gt.shape[0]))]) seld_score[epoch_cnt] = (sed_score[epoch_cnt] + doa_score[epoch_cnt]) / 2 if os.path.isdir('./models'): plot.imshow(conf_mat, cmap='binary', interpolation='None') plot.savefig('models/confusion_matrix.jpg') # New confidence computation, differing doa and sed errors sed_err = sed_loss[epoch_cnt, 0] [sed_conf_low, sed_conf_up] = compute_confidence(sed_err, sed_pred.shape[0]) print("Confidence Interval for SED error is [ %f, %f ]" % (sed_conf_low, sed_conf_up)) doa_err = doa_gt - doa_pred [x_err, y_err, z_err] = compute_doa_confidence(doa_err, n_classes) print('epoch_cnt: %d, time: %.2fs, tr_loss: %.4f, val_loss: %.4f, ' 'F1_overall: %.2f, ER_overall: %.2f, ' 'doa_error_gt: %.2f, doa_error_pred: %.2f, good_pks_ratio:%.2f, ' 'sed_score: %.4f, doa_score: %.4f, seld_score: %.4f' % ( epoch_cnt, time.time() - start, tr_loss[epoch_cnt], val_loss[epoch_cnt], sed_loss[epoch_cnt, 1], sed_loss[epoch_cnt, 0], doa_loss[epoch_cnt, 1], doa_loss[epoch_cnt, 2], doa_loss[epoch_cnt, 5] / float(sed_gt.shape[0]), sed_score[epoch_cnt], doa_score[epoch_cnt], seld_score[epoch_cnt] ) ) simple_plotter.plot_3d("models/doa_gt.txt", "models/doa_pred.txt", 0, 11, 200) simple_plotter.plot_confidence(x_err, y_err, z_err, "ov"+str(ov))
def main(argv): """ Main wrapper for training sound event localization and detection network. :param argv: expects two optional inputs. first input: job_id - (optional) all the output files will be uniquely represented with this. (default) 1 second input: task_id - (optional) To chose the system configuration in parameters.py. (default) uses default parameters """ if len(argv) != 3: print('\n\n') print('-------------------------------------------------------------------------------------------------------') print('The code expected two inputs') print('\t>> python seld.py <job-id> <task-id>') print('\t\t<job-id> is a unique identifier which is used for output filenames (models, training plots). ' 'You can use any number or string for this.') print('\t\t<task-id> is used to choose the user-defined parameter set from parameter.py') print('Using default inputs for now') print('-------------------------------------------------------------------------------------------------------') print('\n\n') # use parameter set defined by user task_id = '1' if len(argv) < 3 else argv[-1] params = parameter.get_params(task_id) job_id = 1 if len(argv) < 2 else argv[1] model_dir = 'models/' utils.create_folder(model_dir) unique_name = '{}_train{}_validation{}_seq{}'.format(params['dataset'], params['train_split'], params['val_split'], params['sequence_length']) unique_name = os.path.join(model_dir, unique_name) print("unique_name: {}\n".format(unique_name)) data_gen_train = cls_data_generator.DataGenerator( dataset=params['dataset'], ov=params['overlap'], split=params['train_split'], db=params['db'], nfft=params['nfft'], batch_size=params['batch_size'], seq_len=params['sequence_length'], classifier_mode=params['mode'], weakness=params['weakness'], datagen_mode='train', cnn3d=params['cnn_3d'], xyz_def_zero=params['xyz_def_zero'], azi_only=params['azi_only'] ) data_gen_test = cls_data_generator.DataGenerator( dataset=params['dataset'], ov=params['overlap'], split=params['val_split'], db=params['db'], nfft=params['nfft'], batch_size=params['batch_size'], seq_len=params['sequence_length'], classifier_mode=params['mode'], weakness=params['weakness'], datagen_mode='validation', cnn3d=params['cnn_3d'], xyz_def_zero=params['xyz_def_zero'], azi_only=params['azi_only'], shuffle=False ) data_in, data_out = data_gen_train.get_data_sizes() #n_classes = data_out[0][2] print( 'FEATURES:\n' '\tdata_in: {}\n' '\tdata_out: {}\n'.format( data_in, data_out ) ) gt = collect_test_labels(data_gen_test, data_out, params['mode'], params['quick_test']) sed_gt = evaluation_metrics.reshape_3Dto2D(gt[0]) doa_gt = evaluation_metrics.reshape_3Dto2D(gt[1]) print( 'MODEL:\n' '\tdropout_rate: {}\n' '\tCNN: nb_cnn_filt: {}, pool_size{}\n' '\trnn_size: {}, fnn_size: {}\n'.format( params['dropout_rate'], params['nb_cnn3d_filt'] if params['cnn_3d'] else params['nb_cnn2d_filt'], params['pool_size'], params['rnn_size'], params['fnn_size'] ) ) model = keras_model.get_model(data_in=data_in, data_out=data_out, dropout_rate=params['dropout_rate'], nb_cnn2d_filt=params['nb_cnn2d_filt'], pool_size=params['pool_size'], rnn_size=params['rnn_size'], fnn_size=params['fnn_size'], classification_mode=params['mode'], weights=params['loss_weights'], summary=True) if (os.path.exists('{}_model.ckpt'.format(unique_name))): print("Model found!") model.load_weights('{}_model.ckpt'.format(unique_name)) for i in range(10): print("###") best_metric = 99999 conf_mat = None best_conf_mat = None best_epoch = -1 patience_cnt = 0 epoch_metric_loss = np.zeros(params['nb_epochs']) sed_score = np.zeros(params['nb_epochs']) doa_score = np.zeros(params['nb_epochs']) seld_score = np.zeros(params['nb_epochs']) tr_loss = np.zeros(params['nb_epochs']) val_loss = np.zeros(params['nb_epochs']) doa_loss = np.zeros((params['nb_epochs'], 6)) sed_loss = np.zeros((params['nb_epochs'], 2)) for epoch_cnt in range(params['nb_epochs']): start = time.time() print("##### Training the model #####") hist = model.fit_generator( generator=data_gen_train.generate(), steps_per_epoch=params['quick_test_steps'] if params[ 'quick_test'] else data_gen_train.get_total_batches_in_data(), validation_data=data_gen_test.generate(), validation_steps=params['quick_test_steps'] if params[ 'quick_test'] else data_gen_test.get_total_batches_in_data(), use_multiprocessing=False, workers=1, epochs=1, verbose=1 ) tr_loss[epoch_cnt] = hist.history.get('loss')[-1] val_loss[epoch_cnt] = hist.history.get('val_loss')[-1] print("##########################") # Save, get model and re-load weights for the predict_generator bug print("##### Saving weights #####") model.save_weights('{}_model.ckpt'.format(unique_name)) model = keras_model.get_model(data_in=data_in, data_out=data_out, dropout_rate=params['dropout_rate'], nb_cnn2d_filt=params['nb_cnn2d_filt'], pool_size=params['pool_size'], rnn_size=params['rnn_size'], fnn_size=params['fnn_size'], classification_mode=params['mode'], weights=params['loss_weights'], summary=False) model.load_weights('{}_model.ckpt'.format(unique_name)) print("##########################") print("#### Prediction on validation split ####") pred = model.predict_generator( generator=data_gen_test.generate(), steps=params['quick_test_steps'] if params['quick_test'] else data_gen_test.get_total_batches_in_data(), use_multiprocessing=False, workers=1, verbose=1 ) print("########################################") # print("pred:",pred[1].shape) if params['mode'] == 'regr': sed_pred = np.array(evaluation_metrics.reshape_3Dto2D(pred[0])) > .5 doa_pred = evaluation_metrics.reshape_3Dto2D(pred[1]) # Old confidence intervals ''' sed_err = sed_gt - sed_pred [sed_conf_low, sed_conf_up, sed_median] = compute_confidence(sed_err) # print("Condidence Interval for SED error is [" + str(sed_conf_low) + ", " + str(sed_conf_up) + "]") print("Confidence Interval for SED error is [ %.5f, %.5f ]" % (sed_conf_low, sed_conf_up)) # print("\tMedian is " + str(sed_median)) print("\tMedian is %.5f" % (sed_median)) # print("\tDisplacement: +/- " + str(sed_conf_up - sed_median)) print("\tDisplacement: +/- %.5f" % (sed_conf_up - sed_median)) doa_err = doa_gt - doa_pred [doa_conf_low, doa_conf_up, doa_median] = compute_confidence(doa_err) # print("Condidence Interval for DOA is [" + str(doa_conf_low) + ", " + str(doa_conf_up) + "]") print("Confidence Interval for DOA is [ %.5f, %.5f ]" % (doa_conf_low, doa_conf_up)) # print("Median is " + str(doa_median)) print("\tMedian is %.5f" % (doa_median)) # print("Displacement: +/- " + str(doa_conf_up - doa_median)) print("\tDisplacement: +/- %.5f" % (doa_conf_up - doa_median)) ''' sed_loss[epoch_cnt, :] = evaluation_metrics.compute_sed_scores(sed_pred, sed_gt, data_gen_test.nb_frames_1s()) if params['azi_only']: doa_loss[epoch_cnt, :], conf_mat = evaluation_metrics.compute_doa_scores_regr_xy(doa_pred, doa_gt, sed_pred, sed_gt) else: doa_loss[epoch_cnt, :], conf_mat = evaluation_metrics.compute_doa_scores_regr_xyz(doa_pred, doa_gt, sed_pred, sed_gt) sed_score[epoch_cnt] = np.mean([sed_loss[epoch_cnt, 0], 1 - sed_loss[epoch_cnt, 1]]) doa_score[epoch_cnt] = np.mean([2 * np.arcsin(doa_loss[epoch_cnt, 1] / 2.0) / np.pi, 1 - (doa_loss[epoch_cnt, 5] / float(doa_gt.shape[0]))]) seld_score[epoch_cnt] = (sed_score[epoch_cnt] + doa_score[epoch_cnt]) / 2 if os.path.isdir('./models'): plot.imshow(conf_mat, cmap='binary', interpolation='None') plot.savefig('models/confusion_matrix.jpg') # New confidence computation, differing doa and sed errors sed_err = sed_loss[epoch_cnt, 0] [sed_conf_low, sed_conf_up] = compute_confidence(sed_err, sed_pred.shape[0]) print("Confidence Interval for SED error is [ %f, %f ]" % (sed_conf_low, sed_conf_up)) #doa_err = doa_gt - doa_pred #[x_err, y_err, z_err] = compute_doa_confidence(doa_err, n_classes) plot_array = [tr_loss[epoch_cnt], # 0 val_loss[epoch_cnt], # 1 sed_loss[epoch_cnt][0], # 2 er sed_loss[epoch_cnt][1], # 3 f1 doa_loss[epoch_cnt][0], # 4 avg_accuracy doa_loss[epoch_cnt][1], # 5 doa_loss_gt doa_loss[epoch_cnt][2], # 6 doa_loss_pred doa_loss[epoch_cnt][3], # 7 doa_loss_gt_cnt doa_loss[epoch_cnt][4], # 8 doa_loss_pred_cnt doa_loss[epoch_cnt][5], # 9 good_frame_cnt sed_score[epoch_cnt], # 10 doa_score[epoch_cnt], seld_score[epoch_cnt], #doa_conf_low, doa_median, #doa_conf_up, sed_conf_low, #sed_median, sed_conf_up] sed_conf_low, sed_conf_up] patience_cnt += 1 # model.save_weights('{}_model.ckpt'.format(unique_name)) simple_plotter.save_array_to_csv("{}_plot.csv".format(unique_name), plot_array) #simple_plotter.plot_confidence(x_err, y_err, z_err, "ov") print("##### Model and metrics saved! #####") if seld_score[epoch_cnt] < best_metric: best_metric = seld_score[epoch_cnt] best_conf_mat = conf_mat best_epoch = epoch_cnt # Now we save the model at every iteration model.save_weights('{}_BEST_model.ckpt'.format(unique_name)) patience_cnt = 0 print('epoch_cnt: %d, time: %.2fs, tr_loss: %.4f, val_loss: %.4f, ' 'F1_overall: %.2f, ER_overall: %.2f, ' 'doa_error_gt: %.2f, doa_error_pred: %.2f, good_pks_ratio:%.2f, ' 'sed_score: %.4f, doa_score: %.4f, seld_score: %.4f, best_error_metric: %.2f, best_epoch : %d' % ( epoch_cnt, time.time() - start, tr_loss[epoch_cnt], val_loss[epoch_cnt], sed_loss[epoch_cnt, 1], sed_loss[epoch_cnt, 0], doa_loss[epoch_cnt, 1], doa_loss[epoch_cnt, 2], doa_loss[epoch_cnt, 5] / float(sed_gt.shape[0]), sed_score[epoch_cnt], doa_score[epoch_cnt], seld_score[epoch_cnt], best_metric, best_epoch ) ) # plot_functions(unique_name, tr_loss, val_loss, sed_loss, doa_loss, sed_score, doa_score, epoch_cnt) print('best_conf_mat : {}'.format(best_conf_mat)) print('best_conf_mat_diag : {}'.format(np.diag(best_conf_mat))) print('saved model for the best_epoch: {} with best_metric: {}, '.format(best_epoch, best_metric)) print('DOA Metrics: doa_loss_gt: {}, doa_loss_pred: {}, good_pks_ratio: {}'.format( doa_loss[best_epoch, 1], doa_loss[best_epoch, 2], doa_loss[best_epoch, 5] / float(sed_gt.shape[0]))) print('SED Metrics: ER_overall: {}, F1_overall: {}'.format(sed_loss[best_epoch, 0], sed_loss[best_epoch, 1])) print('unique_name: {} '.format(unique_name))
def evaluate(model, data_gen_test, params, log_dir=".", unique_name="unique_name"): logger.info("EVALUATE function called") sed_gt, doa_gt = collect_ground_truth(data_gen_test, params) predict_single_batch(model, data_gen_test) dnn_output = model.predict( x=data_gen_test.generate(), steps=data_gen_test.get_total_batches_in_data(), verbose=2 ) sed_pred = dnn_output[0] > 0.5 doa_pred = dnn_output[1] sed_pred = evaluation_metrics.reshape_3Dto2D(sed_pred) doa_pred = evaluation_metrics.reshape_3Dto2D(doa_pred) num_classes = sed_pred.shape[-1] num_dims_xyz = 3 if doa_pred.shape[-1] > num_classes * num_dims_xyz: # true means we are using masked mse sed_mask = np.repeat(sed_pred, 3, -1) doa_pred = doa_pred[..., num_classes:] * sed_mask doa_gt = doa_gt[..., num_classes:] * sed_mask sed_loss = evaluation_metrics.compute_sed_scores(sed_pred, sed_gt, data_gen_test.nb_frames_1s()) if params['azi_only']: doa_loss, conf_mat = evaluation_metrics.compute_doa_scores_regr_xy(doa_pred, doa_gt, sed_pred, sed_gt) else: doa_loss, conf_mat = evaluation_metrics.compute_doa_scores_regr_xyz(doa_pred, doa_gt, sed_pred, sed_gt) epoch_metric_loss = np.mean([ sed_loss[0], 1 - sed_loss[1], 2 * np.arcsin(doa_loss[1] / 2.0) / np.pi, 1 - (doa_loss[5] / float(doa_gt.shape[0]))] ) logger.info( 'F1_overall: %.2f, ER_overall: %.2f, ' 'doa_error_gt: %.2f, doa_error_pred: %.2f, good_pks_ratio:%.2f, ' 'error_metric: %.2f' % ( sed_loss[1], sed_loss[0], doa_loss[1], doa_loss[2], doa_loss[5] / float(sed_gt.shape[0]), epoch_metric_loss ) ) logger.info('DOA Metrics: doa_loss_gt: {}, doa_loss_pred: {}, good_pks_ratio: {}'.format( doa_loss[1], doa_loss[2], doa_loss[5] / float(sed_gt.shape[0]))) logger.info( 'SED Metrics: F1_overall: {}, ER_overall: {}'.format(sed_loss[1], sed_loss[0])) logger.info('unique_name: {} '.format(unique_name))
def train(model, data_gen_train, data_gen_val, params, log_dir=".", unique_name="unique_name"): logger.info("Train function called") sed_gt, doa_gt = collect_ground_truth(data_gen_val, params) best_metric = 99999 conf_mat = None best_conf_mat = None best_epoch = -1 patience_cnt = 0 epoch_metric_loss = np.zeros(params['nb_epochs']) tr_loss = np.zeros(params['nb_epochs']) val_loss = np.zeros(params['nb_epochs']) doa_loss = np.zeros((params['nb_epochs'], 6)) sed_loss = np.zeros((params['nb_epochs'], 2)) nb_epoch = params['nb_epochs'] model_path = os.path.join(log_dir, 'model') K.clear_session() for epoch_cnt in range(nb_epoch): logger.info(f"Iteration {epoch_cnt}/{nb_epoch}") start = time.time() hist = model.fit( x=data_gen_train.generate(), steps_per_epoch=data_gen_train.get_total_batches_in_data(), validation_data=data_gen_val.generate(), validation_steps=data_gen_val.get_total_batches_in_data(), epochs=params['epochs_per_iteration'], verbose=2, # callbacks=[MyCustomCallback] ) tr_loss[epoch_cnt] = hist.history.get('loss')[-1] val_loss[epoch_cnt] = hist.history.get('val_loss')[-1] if (params['debug_load_single_batch']) and (epoch_cnt % 10 != 0) and (epoch_cnt != nb_epoch - 1): plot_functions(os.path.join(log_dir, 'training_curves'), tr_loss, val_loss, sed_loss, doa_loss, epoch_metric_loss) else: predict_single_batch(model, data_gen_train) pred = model.predict( x=data_gen_val.generate(), steps=data_gen_val.get_total_batches_in_data(), verbose=2 ) if params['mode'] == 'regr': sed_pred = evaluation_metrics.reshape_3Dto2D(pred[0]) > 0.5 doa_pred = pred[1] num_classes = sed_pred.shape[-1] num_dims_xyz = 3 if doa_pred.shape[-1] > num_classes * num_dims_xyz: # true means we are using masked mse doa_pred = doa_pred[..., num_classes:] logger.debug(f"doa_pred.shape {doa_pred.shape}") if doa_gt.shape[-1] > num_classes * num_dims_xyz: # true means we are using masked mse doa_gt = doa_gt[..., num_classes:] logger.debug(f"doa_gt.shape {doa_gt.shape}") doa_pred = evaluation_metrics.reshape_3Dto2D(doa_pred) sed_loss[epoch_cnt, :] = evaluation_metrics.compute_sed_scores(sed_pred, sed_gt, data_gen_val.nb_frames_1s()) if params['azi_only']: doa_loss[epoch_cnt, :], conf_mat = evaluation_metrics.compute_doa_scores_regr_xy(doa_pred, doa_gt, sed_pred, sed_gt) else: doa_loss[epoch_cnt, :], conf_mat = evaluation_metrics.compute_doa_scores_regr_xyz(doa_pred, doa_gt, sed_pred, sed_gt) epoch_metric_loss[epoch_cnt] = np.mean([ sed_loss[epoch_cnt, 0], 1 - sed_loss[epoch_cnt, 1], 2 * np.arcsin(doa_loss[epoch_cnt, 1] / 2.0) / np.pi, 1 - (doa_loss[epoch_cnt, 5] / float(doa_gt.shape[0]))] ) plot_functions(os.path.join(log_dir, 'training_curves'), tr_loss, val_loss, sed_loss, doa_loss, epoch_metric_loss) patience_cnt += 1 if (epoch_metric_loss[epoch_cnt] < best_metric): best_metric = epoch_metric_loss[epoch_cnt] best_conf_mat = conf_mat best_epoch = epoch_cnt model.save(model_path) patience_cnt = 0 logger.info( 'epoch_cnt: %d, time: %.2fs, tr_loss: %.2f, val_loss: %.2f, ' 'F1_overall: %.2f, ER_overall: %.2f, ' 'doa_error_gt: %.2f, doa_error_pred: %.2f, good_pks_ratio:%.2f, ' 'error_metric: %.2f, best_error_metric: %.2f, best_epoch : %d' % ( epoch_cnt, time.time() - start, tr_loss[epoch_cnt], val_loss[epoch_cnt], sed_loss[epoch_cnt, 1], sed_loss[epoch_cnt, 0], doa_loss[epoch_cnt, 1], doa_loss[epoch_cnt, 2], doa_loss[epoch_cnt, 5] / float(sed_gt.shape[0]), epoch_metric_loss[epoch_cnt], best_metric, best_epoch ) ) if patience_cnt > params['patience']: break # otherwise RAM use increases after every epoch. But is the optimizer state forgotten? K.clear_session() if params['debug_load_single_batch'] and hist.history.get('loss')[-1] < 0.01: break if params['debug_load_single_batch']: model.save(model_path) else: logger.info('best_conf_mat : {}'.format(best_conf_mat)) logger.info('best_conf_mat_diag : {}'.format(np.diag(best_conf_mat))) logger.info('saved model for the best_epoch: {} with best_metric: {}, '.format(best_epoch, best_metric)) logger.info('DOA Metrics: doa_loss_gt: {}, doa_loss_pred: {}, good_pks_ratio: {}'.format( doa_loss[best_epoch, 1], doa_loss[best_epoch, 2], doa_loss[best_epoch, 5] / float(sed_gt.shape[0]))) logger.info( 'SED Metrics: F1_overall: {}, ER_overall: {}'.format(sed_loss[best_epoch, 1], sed_loss[best_epoch, 0])) np.save(os.path.join(log_dir, 'training-loss'), [tr_loss, val_loss]) logger.info(f'unique_name: {unique_name}') logger.info(f'log_dir: {log_dir}')
def main(args): """ Main wrapper for training sound event localization and detection network. :param argv: expects two optional inputs. first input: job_id - (optional) all the output files will be uniquely represented with this. (default) 1 second input: task_id - (optional) To chose the system configuration in parameters.py. (default) uses default parameters """ # use parameter set defined by user task_id = args.params params = parameter.get_params(task_id) job_id = args.model_name model_dir = 'models/' + args.author + '/' if args.author != "" else 'models/' utils.create_folder(model_dir) unique_name = '{}_ov{}_split{}_{}{}_3d{}_{}'.format( params['dataset'], params['overlap'], params['split'], params['mode'], params['weakness'], int(params['cnn_3d']), job_id) model_name = unique_name epoch_manager = JSON_Manager(args.author, unique_name) logdir = "logs/" + args.author + "/" + unique_name unique_name = os.path.join(model_dir, unique_name) print("unique_name: {}\n".format(unique_name)) data_gen_test = cls_data_generator_seld.DataGenerator( dataset=params['dataset'], ov=params['overlap'], split=params['split'], db=params['db'], nfft=params['nfft'], batch_size=params['batch_size'], seq_len=params['sequence_length'], classifier_mode=params['mode'], weakness=params['weakness'], datagen_mode='test', cnn3d=params['cnn_3d'], xyz_def_zero=params['xyz_def_zero'], azi_only=params['azi_only'], shuffle=False) data_in, data_out = data_gen_test.get_data_sizes() print('FEATURES:\n' '\tdata_in: {}\n' '\tdata_out: {}\n'.format(data_in, data_out)) gt = collect_test_labels(data_gen_test, data_out, params['mode'], params['quick_test']) sed_gt = evaluation_metrics.reshape_3Dto2D(gt[0]) doa_gt = evaluation_metrics.reshape_3Dto2D(gt[1]) print('MODEL:\n' '\tdropout_rate: {}\n' '\tCNN: nb_cnn_filt: {}, pool_size{}\n' '\trnn_size: {}, fnn_size: {}\n'.format( params['dropout_rate'], params['nb_cnn3d_filt'] if params['cnn_3d'] else params['nb_cnn2d_filt'], params['pool_size'], params['rnn_size'], params['fnn_size'])) model = load_model(os.path.join(model_dir, model_name + "_best_model.h5"), custom_objects={ 'QuaternionConv2D': QuaternionConv2D, 'QuaternionGRU': QuaternionGRU, 'QuaternionDense': QuaternionDense }) model.summary() plot_model(model, to_file=os.path.join(model_dir, 'model.png')) best_metric = epoch_manager.get_best_metric() conf_mat = None best_conf_mat = epoch_manager.get_best_conf_mat() best_epoch = epoch_manager.get_best_epoch() patience_cnt = epoch_manager.get_patience_cnt() epoch_metric_loss = np.zeros(params['nb_epochs']) sed_score = np.zeros(params['nb_epochs']) std_score = np.zeros(params['nb_epochs']) doa_score = np.zeros(params['nb_epochs']) seld_score = np.zeros(params['nb_epochs']) tr_loss = np.zeros(params['nb_epochs']) val_loss = np.zeros(params['nb_epochs']) doa_loss = np.zeros((params['nb_epochs'], 6)) sed_loss = np.zeros((params['nb_epochs'], 2)) epoch_cnt = 0 pred = model.predict_generator( generator=data_gen_test.generate(), steps=data_gen_test.get_total_batches_in_data(), use_multiprocessing=False, verbose=2) print("pred[1]:", pred[1].shape) if params['mode'] == 'regr': sed_pred = evaluation_metrics.reshape_3Dto2D(pred[0]) > 0.5 print(f"sed_pred: {sed_pred.shape}") doa_pred = evaluation_metrics.reshape_3Dto2D(pred[1]) print(f"doa_pred: {doa_pred.shape}") sed_loss[epoch_cnt, :] = evaluation_metrics.compute_sed_scores( sed_pred, sed_gt, data_gen_test.nb_frames_1s()) if params['azi_only']: doa_loss[ epoch_cnt, :], conf_mat = evaluation_metrics.compute_doa_scores_regr_xy( doa_pred, doa_gt, sed_pred, sed_gt) else: doa_loss[ epoch_cnt, :], conf_mat = evaluation_metrics.compute_doa_scores_regr_xyz( doa_pred, doa_gt, sed_pred, sed_gt) sed_score[epoch_cnt] = np.mean( [sed_loss[epoch_cnt, 0], 1 - sed_loss[epoch_cnt, 1]]) print(f"ER: {sed_loss[epoch_cnt, 0]}") er = sed_loss[epoch_cnt, 0] interval = 1.96 * np.sqrt(((er) * (1 - er)) / sed_pred.shape[0]) print(f"interval: {interval}") doa_score[epoch_cnt] = np.mean([ 2 * np.arcsin(doa_loss[epoch_cnt, 1] / 2.0) / np.pi, 1 - (doa_loss[epoch_cnt, 5] / float(doa_gt.shape[0])) ]) seld_score[epoch_cnt] = np.mean( [sed_score[epoch_cnt], doa_score[epoch_cnt]]) doa_error = doa_pred - doa_gt doa_error = np.reshape(doa_error, newshape=(doa_error.shape[0], 11, 2)) doa_error = np.absolute(doa_error[:, :, 0]) print(f"doa_error: {doa_error.shape}") doa_error = np.reshape(doa_error, newshape=(doa_error.shape[0] * doa_error.shape[1])) print(f"doa_error: {doa_error.shape}") np.save(model_name + "_x", doa_error) doa_error = doa_pred - doa_gt doa_error = np.reshape(doa_error, newshape=(doa_error.shape[0], 11, 2)) doa_error = np.absolute(doa_error[:, :, 1]) print(f"doa_error: {doa_error.shape}") doa_error = np.reshape(doa_error, newshape=(doa_error.shape[0] * doa_error.shape[1])) print(f"doa_error: {doa_error.shape}") np.save(model_name + "_y", doa_error) # standard deviation std_score[epoch_cnt] = np.std( [sed_score[epoch_cnt], doa_score[epoch_cnt]]) print(f"{er-interval} / {er+interval}") #lower_confidence, upper_confidence = evaluation_metrics.compute_confidence_interval(best_metric,best_std, params['nb_epochs'], confid_coeff=1.96) # 1.96 for a 95% CI print("\n---- FINISHED ----\n")