def do_job(iter_num): model = get_model() x = [12345, 54321, 00000, 1111] y = [1, 1, 0, 0] model.fit(x=x, y=y, validation_split=0.5, epochs=10, shuffle=True, batch_size=2, verbose=2) print("Done process " + str(iter_num)) clear_session() print("close")
def main(argv): """ Main wrapper for training sound event localization and detection network. :param argv: expects two optional inputs. first input: job_id - (optional) all the output files will be uniquely represented with this. (default) 1 second input: task_id - (optional) To chose the system configuration in parameters.py. (default) uses default parameters """ if len(argv) != 3: print('\n\n') print('-------------------------------------------------------------------------------------------------------') print('The code expected two inputs') print('\t>> python seld.py <job-id> <task-id>') print('\t\t<job-id> is a unique identifier which is used for output filenames (models, training plots). ' 'You can use any number or string for this.') print('\t\t<task-id> is used to choose the user-defined parameter set from parameter.py') print('Using default inputs for now') print('-------------------------------------------------------------------------------------------------------') print('\n\n') # use parameter set defined by user task_id = '1' if len(argv) < 3 else argv[-1] params = parameter.get_params(task_id) job_id = 1 if len(argv) < 2 else argv[1] model_dir = 'models/' utils.create_folder(model_dir) unique_name = '{}_ov{}_split{}_{}{}_3d{}_{}'.format( params['dataset'], params['overlap'], params['split'], params['mode'], params['weakness'], int(params['cnn_3d']), job_id ) unique_name = os.path.join(model_dir, unique_name) print("unique_name: {}\n".format(unique_name)) data_gen_train = cls_data_generator.DataGenerator( dataset=params['dataset'], ov=params['overlap'], split=params['split'], db=params['db'], nfft=params['nfft'], batch_size=params['batch_size'], seq_len=params['sequence_length'], classifier_mode=params['mode'], weakness=params['weakness'], datagen_mode='train', cnn3d=params['cnn_3d'], xyz_def_zero=params['xyz_def_zero'], azi_only=params['azi_only'] ) data_gen_test = cls_data_generator.DataGenerator( dataset=params['dataset'], ov=params['overlap'], split=params['split'], db=params['db'], nfft=params['nfft'], batch_size=params['batch_size'], seq_len=params['sequence_length'], classifier_mode=params['mode'], weakness=params['weakness'], datagen_mode='test', cnn3d=params['cnn_3d'], xyz_def_zero=params['xyz_def_zero'], azi_only=params['azi_only'], shuffle=False ) data_in, data_out = data_gen_train.get_data_sizes() print( 'FEATURES:\n' '\tdata_in: {}\n' '\tdata_out: {}\n'.format( data_in, data_out ) ) gt = collect_test_labels(data_gen_test, data_out, params['mode'], params['quick_test']) sed_gt = evaluation_metrics.reshape_3Dto2D(gt[0]) doa_gt = evaluation_metrics.reshape_3Dto2D(gt[1]) print( 'MODEL:\n' '\tdropout_rate: {}\n' '\tCNN: nb_cnn_filt: {}, pool_size{}\n' '\trnn_size: {}, fnn_size: {}\n'.format( params['dropout_rate'], params['nb_cnn3d_filt'] if params['cnn_3d'] else params['nb_cnn2d_filt'], params['pool_size'], params['rnn_size'], params['fnn_size'] ) ) model = keras_model.get_model(data_in=data_in, data_out=data_out, dropout_rate=params['dropout_rate'], nb_cnn2d_filt=params['nb_cnn2d_filt'], pool_size=params['pool_size'], rnn_size=params['rnn_size'], fnn_size=params['fnn_size'], classification_mode=params['mode'], weights=params['loss_weights'], loader=False, loader2=False) # Change loader to True to enable transfer learning, Change loader2 to True to enable transfer learning with different labels best_metric = 99999 conf_mat = None best_conf_mat = None best_epoch = -1 patience_cnt = 0 epoch_metric_loss = np.zeros(params['nb_epochs']) tr_loss = np.zeros(params['nb_epochs']) val_loss = np.zeros(params['nb_epochs']) doa_loss = np.zeros((params['nb_epochs'], 6)) sed_loss = np.zeros((params['nb_epochs'], 2)) nb_epoch = 2 if params['quick_test'] else params['nb_epochs'] for epoch_cnt in range(nb_epoch): start = time.time() hist = model.fit_generator( generator=data_gen_train.generate(), steps_per_epoch=2 if params['quick_test'] else data_gen_train.get_total_batches_in_data(), validation_data=data_gen_test.generate(), validation_steps=2 if params['quick_test'] else data_gen_test.get_total_batches_in_data(), epochs=1, verbose=0 ) tr_loss[epoch_cnt] = hist.history.get('loss')[-1] val_loss[epoch_cnt] = hist.history.get('val_loss')[-1] pred = model.predict_generator( generator=data_gen_test.generate(), steps=2 if params['quick_test'] else data_gen_test.get_total_batches_in_data(), verbose=2 ) if params['mode'] == 'regr': sed_pred = evaluation_metrics.reshape_3Dto2D(pred[0]) > 0.5 doa_pred = evaluation_metrics.reshape_3Dto2D(pred[1]) sed_loss[epoch_cnt, :] = evaluation_metrics.compute_sed_scores(sed_pred, sed_gt, data_gen_test.nb_frames_1s()) if params['azi_only']: doa_loss[epoch_cnt, :], conf_mat = evaluation_metrics.compute_doa_scores_regr_xy(doa_pred, doa_gt, sed_pred, sed_gt) else: doa_loss[epoch_cnt, :], conf_mat = evaluation_metrics.compute_doa_scores_regr_xyz(doa_pred, doa_gt, sed_pred, sed_gt) epoch_metric_loss[epoch_cnt] = np.mean([ sed_loss[epoch_cnt, 0], 1-sed_loss[epoch_cnt, 1], 2*np.arcsin(doa_loss[epoch_cnt, 1]/2.0)/np.pi, 1 - (doa_loss[epoch_cnt, 5] / float(doa_gt.shape[0]))] ) plot_functions(unique_name, tr_loss, val_loss, sed_loss, doa_loss, epoch_metric_loss) patience_cnt += 1 if epoch_metric_loss[epoch_cnt] < best_metric: best_metric = epoch_metric_loss[epoch_cnt] best_conf_mat = conf_mat best_epoch = epoch_cnt model.save('{}_model.h5'.format(unique_name)) patience_cnt = 0 print( 'epoch_cnt: %d, time: %.2fs, tr_loss: %.2f, val_loss: %.2f, ' 'F1_overall: %.2f, ER_overall: %.2f, ' 'doa_error_gt: %.2f, doa_error_pred: %.2f, good_pks_ratio:%.2f, ' 'error_metric: %.2f, best_error_metric: %.2f, best_epoch : %d' % ( epoch_cnt, time.time() - start, tr_loss[epoch_cnt], val_loss[epoch_cnt], sed_loss[epoch_cnt, 1], sed_loss[epoch_cnt, 0], doa_loss[epoch_cnt, 1], doa_loss[epoch_cnt, 2], doa_loss[epoch_cnt, 5] / float(sed_gt.shape[0]), epoch_metric_loss[epoch_cnt], best_metric, best_epoch ) ) if patience_cnt > params['patience']: break print('best_conf_mat : {}'.format(best_conf_mat)) print('best_conf_mat_diag : {}'.format(np.diag(best_conf_mat))) print('saved model for the best_epoch: {} with best_metric: {}, '.format(best_epoch, best_metric)) print('DOA Metrics: doa_loss_gt: {}, doa_loss_pred: {}, good_pks_ratio: {}'.format( doa_loss[best_epoch, 1], doa_loss[best_epoch, 2], doa_loss[best_epoch, 5] / float(sed_gt.shape[0]))) print('SED Metrics: ER_overall: {}, F1_overall: {}'.format(sed_loss[best_epoch, 1], sed_loss[best_epoch, 0])) print('unique_name: {} '.format(unique_name))
data = file_list_to_data(files, msg="generate train_dataset", n_mels=param["feature"]["n_mels"], n_frames=param["feature"]["n_frames"], n_hop_frames=param["feature"]["n_hop_frames"], n_fft=param["feature"]["n_fft"], hop_length=param["feature"]["hop_length"], power=param["feature"]["power"]) # number of vectors for each wave file n_vectors_ea_file = int(data.shape[0] / len(files)) # train model print("============== MODEL TRAINING ==============") model = keras_model.get_model(param["feature"]["n_mels"] * param["feature"]["n_frames"], param["fit"]["lr"]) model.summary() history = model.fit(x=data, y=data, epochs=param["fit"]["epochs"], batch_size=param["fit"]["batch_size"], shuffle=param["fit"]["shuffle"], validation_split=param["fit"]["validation_split"], verbose=param["fit"]["verbose"]) # calculate y_pred for fitting anomaly score distribution y_pred = [] start_idx = 0 for file_idx in range(len(files)):
import numpy as np import os import argparse import keras_model as models import get_dataset as aww_data import aww_util num_classes = 12 # should probably draw this directly from the dataset. # FLAGS = None if __name__ == '__main__': Flags, unparsed = aww_util.parse_command() print('We will download data to {:}'.format(Flags.data_dir)) print('We will train for {:} epochs'.format(Flags.epochs)) ds_train, ds_test, ds_val = aww_data.get_training_data(Flags) print("Done getting data") model = models.get_model(model_name=Flags.model_architecture) model.summary() train_hist = model.fit(ds_train, validation_data=ds_val, epochs=Flags.epochs) model.save(Flags.saved_model_path) if Flags.run_test_set: test_scores = model.evaluate(ds_test) print("Test loss:", test_scores[0]) print("Test accuracy:", test_scores[1])
def main(argv): """ Main wrapper for training sound event localization and detection network. :param argv: expects two optional inputs. first input: task_id - (optional) To chose the system configuration in parameters.py. (default) 1 - uses default parameters second input: job_id - (optional) all the output files will be uniquely represented with this. (default) 1 """ print(argv) if len(argv) != 3: print('\n\n') print( '-------------------------------------------------------------------------------------------------------' ) print('The code expected two optional inputs') print('\t>> python seld.py <task-id> <job-id>') print( '\t\t<task-id> is used to choose the user-defined parameter set from parameter.py' ) print('Using default inputs for now') print( '\t\t<job-id> is a unique identifier which is used for output filenames (models, training plots). ' 'You can use any number or string for this.') print( '-------------------------------------------------------------------------------------------------------' ) print('\n\n') # use parameter set defined by user task_id = '1' if len(argv) < 2 else argv[1] params = parameter.get_params(task_id) job_id = 1 if len(argv) < 3 else argv[-1] feat_cls = cls_feature_class.FeatureClass(params) train_splits, val_splits, test_splits = None, None, None if params['mode'] == 'dev': test_splits = [6] val_splits = [5] train_splits = [[1, 2, 3, 4]] elif params['mode'] == 'eval': test_splits = [[7, 8]] val_splits = [[6]] train_splits = [[1, 2, 3, 4, 5]] for split_cnt, split in enumerate(test_splits): print( '\n\n---------------------------------------------------------------------------------------------------' ) print( '------------------------------------ SPLIT {} -----------------------------------------------' .format(split)) print( '---------------------------------------------------------------------------------------------------' ) # Unique name for the run cls_feature_class.create_folder(params['model_dir']) unique_name = '{}_{}_{}_{}_split{}'.format(task_id, job_id, params['dataset'], params['mode'], split) unique_name = os.path.join(params['model_dir'], unique_name) model_name = '{}_model.h5'.format(unique_name) print("unique_name: {}\n".format(unique_name)) # Load train and validation data print('Loading training dataset:') data_gen_train = cls_data_generator.DataGenerator( params=params, split=train_splits[split_cnt]) print('Loading validation dataset:') data_gen_val = cls_data_generator.DataGenerator( params=params, split=val_splits[split_cnt], shuffle=False, per_file=True, is_eval=False) # Collect the reference labels for validation data data_in, data_out = data_gen_train.get_data_sizes() print('FEATURES:\n\tdata_in: {}\n\tdata_out: {}\n'.format( data_in, data_out)) nb_classes = data_gen_train.get_nb_classes() print( 'MODEL:\n\tdropout_rate: {}\n\tCNN: nb_cnn_filt: {}, f_pool_size{}, t_pool_size{}\n\trnn_size: {}, fnn_size: {}\n\tdoa_objective: {}\n' .format(params['dropout_rate'], params['nb_cnn2d_filt'], params['f_pool_size'], params['t_pool_size'], params['rnn_size'], params['fnn_size'], params['doa_objective'])) print('Using loss weights : {}'.format(params['loss_weights'])) model = keras_model.get_model(data_in=data_in, data_out=data_out, dropout_rate=params['dropout_rate'], nb_cnn2d_filt=params['nb_cnn2d_filt'], f_pool_size=params['f_pool_size'], t_pool_size=params['t_pool_size'], rnn_size=params['rnn_size'], fnn_size=params['fnn_size'], weights=params['loss_weights'], doa_objective=params['doa_objective'], is_accdoa=params['is_accdoa']) # Dump results in DCASE output format for calculating final scores dcase_output_val_folder = os.path.join( params['dcase_output_dir'], '{}_{}_{}_val'.format(task_id, params['dataset'], params['mode'])) cls_feature_class.delete_and_create_folder(dcase_output_val_folder) print('Dumping recording-wise val results in: {}'.format( dcase_output_val_folder)) # Initialize evaluation metric class score_obj = ComputeSELDResults(params) best_seld_metric = 99999 best_epoch = -1 patience_cnt = 0 nb_epoch = 2 if params['quick_test'] else params['nb_epochs'] tr_loss = np.zeros(nb_epoch) seld_metric = np.zeros((nb_epoch, 5)) # start training for epoch_cnt in range(nb_epoch): start = time.time() # train once per epoch hist = model.fit_generator( generator=data_gen_train.generate(), steps_per_epoch=2 if params['quick_test'] else data_gen_train.get_total_batches_in_data(), epochs=params['epochs_per_fit'], verbose=2, ) tr_loss[epoch_cnt] = hist.history.get('loss')[-1] # predict once per epoch pred = model.predict_generator( generator=data_gen_val.generate(), steps=2 if params['quick_test'] else data_gen_val.get_total_batches_in_data(), verbose=2) if params['is_accdoa']: sed_pred, doa_pred = get_accdoa_labels(pred, nb_classes) sed_pred = reshape_3Dto2D(sed_pred) doa_pred = reshape_3Dto2D(doa_pred) else: sed_pred = reshape_3Dto2D(pred[0]) > 0.5 doa_pred = reshape_3Dto2D(pred[1] if params['doa_objective'] is 'mse' else pred[1][:, :, nb_classes:]) # Calculate the DCASE 2021 metrics - Location-aware detection and Class-aware localization scores dump_DCASE2021_results(data_gen_val, feat_cls, dcase_output_val_folder, sed_pred, doa_pred) seld_metric[epoch_cnt, :] = score_obj.get_SELD_Results( dcase_output_val_folder) patience_cnt += 1 if seld_metric[epoch_cnt, -1] < best_seld_metric: best_seld_metric = seld_metric[epoch_cnt, -1] best_epoch = epoch_cnt model.save(model_name) patience_cnt = 0 print( 'epoch_cnt: {}, time: {:0.2f}s, tr_loss: {:0.2f}, ' '\n\t\t DCASE2021 SCORES: ER: {:0.2f}, F: {:0.1f}, LE: {:0.1f}, LR:{:0.1f}, seld_score (early stopping score): {:0.2f}, ' 'best_seld_score: {:0.2f}, best_epoch : {}\n'.format( epoch_cnt, time.time() - start, tr_loss[epoch_cnt], seld_metric[epoch_cnt, 0], seld_metric[epoch_cnt, 1] * 100, seld_metric[epoch_cnt, 2], seld_metric[epoch_cnt, 3] * 100, seld_metric[epoch_cnt, -1], best_seld_metric, best_epoch)) if patience_cnt > params['patience']: break print('\nResults on validation split:') print('\tUnique_name: {} '.format(unique_name)) print('\tSaved model for the best_epoch: {}'.format(best_epoch)) print('\tSELD_score (early stopping score) : {}'.format( best_seld_metric)) print('\n\tDCASE2021 scores') print( '\tClass-aware localization scores: Localization Error: {:0.1f}, Localization Recall: {:0.1f}' .format(seld_metric[best_epoch, 2], seld_metric[best_epoch, 3] * 100)) print( '\tLocation-aware detection scores: Error rate: {:0.2f}, F-score: {:0.1f}' .format(seld_metric[best_epoch, 0], seld_metric[best_epoch, 1] * 100)) # ------------------ Calculate metric scores for unseen test split --------------------------------- print( '\nLoading the best model and predicting results on the testing split' ) print('\tLoading testing dataset:') data_gen_test = cls_data_generator.DataGenerator( params=params, split=split, shuffle=False, per_file=True, is_eval=True if params['mode'] is 'eval' else False) model = keras_model.load_seld_model('{}_model.h5'.format(unique_name), params['doa_objective']) pred_test = model.predict_generator( generator=data_gen_test.generate(), steps=2 if params['quick_test'] else data_gen_test.get_total_batches_in_data(), verbose=2) if params['is_accdoa']: test_sed_pred, test_doa_pred = get_accdoa_labels( pred_test, nb_classes) test_sed_pred = reshape_3Dto2D(test_sed_pred) test_doa_pred = reshape_3Dto2D(test_doa_pred) else: test_sed_pred = reshape_3Dto2D(pred_test[0]) > 0.5 test_doa_pred = reshape_3Dto2D( pred_test[1] if params['doa_objective'] is 'mse' else pred_test[1][:, :, nb_classes:]) # Dump results in DCASE output format for calculating final scores dcase_output_test_folder = os.path.join( params['dcase_output_dir'], '{}_{}_{}_test'.format(task_id, params['dataset'], params['mode'])) cls_feature_class.delete_and_create_folder(dcase_output_test_folder) print('Dumping recording-wise test results in: {}'.format( dcase_output_test_folder)) dump_DCASE2021_results(data_gen_test, feat_cls, dcase_output_test_folder, test_sed_pred, test_doa_pred) if params['mode'] is 'dev': # Calculate DCASE2021 scores test_seld_metric = score_obj.get_SELD_Results( dcase_output_test_folder) print('Results on test split:') print('\tDCASE2021 Scores') print( '\tClass-aware localization scores: Localization Error: {:0.1f}, Localization Recall: {:0.1f}' .format(test_seld_metric[2], test_seld_metric[3] * 100)) print( '\tLocation-aware detection scores: Error rate: {:0.2f}, F-score: {:0.1f}' .format(test_seld_metric[0], test_seld_metric[1] * 100)) print('\tSELD (early stopping metric): {:0.2f}'.format( test_seld_metric[-1]))
# generate dataset print("============== DATASET_GENERATOR ==============") files = file_list_generator(target_dir) train_data = list_to_vector_array( files, msg="generate train_dataset", n_mels=param["feature"]["n_mels"], frames=param["feature"]["frames"], n_fft=param["feature"]["n_fft"], hop_length=param["feature"]["hop_length"], power=param["feature"]["power"]) # train model print("============== MODEL TRAINING ==============") model = keras_model.get_model(param["feature"]["n_mels"] * param["feature"]["frames"]) model.summary() model.compile(**param["fit"]["compile"]) history = model.fit(train_data, train_data, epochs=param["fit"]["epochs"], batch_size=param["fit"]["batch_size"], shuffle=param["fit"]["shuffle"], validation_split=param["fit"]["validation_split"], verbose=param["fit"]["verbose"]) visualizer.loss_plot(history.history["loss"], history.history["val_loss"]) visualizer.save_figure(history_img) model.save(model_file_path)
# train model print("============== MODEL TRAINING ==============") # checkpoint model_checkpoint = ModelCheckpoint(best_model_filepath + "{epoch:02d}.hdf5", monitor='val_loss', verbose=1, save_best_only=True) early = EarlyStopping(monitor='val_loss', mode='min', patience=10, min_delta=0.0001) # create model model = keras_model.get_model((shape0_feat, shape1_feat), param["autoencoder"]["latentDim"]) model.summary() #train model model.compile(**param["fit"]["compile"]) history = model.fit_generator(gen_train, validation_data=gen_val, epochs=param["fit"]["epochs"], verbose=param["fit"]["verbose"], callbacks=[model_checkpoint, early]) visualizer.loss_plot(history.history["loss"], history.history["val_loss"]) visualizer.save_figure(history_img) model.save(model_file_path) com.logger.info("save_model -> {}".format(model_file_path))
num_classes = 12 # should probably draw this directly from the dataset. # FLAGS = None if __name__ == '__main__': Flags, unparsed = kws_util.parse_command() print('We will download data to {:}'.format(Flags.data_dir)) ds_train, ds_test, ds_val = kws_data.get_training_data(Flags) print("Done getting data") if Flags.model_init_path is None: print( "Starting with untrained model. Accuracy will be random-guess level at best." ) model = models.get_model(args=Flags) else: print(f"Starting with pre-trained model from {Flags.model_init_path}") model = keras.models.load_model(Flags.model_init_path) model.summary() test_scores = model.evaluate(ds_test, return_dict=True) print("Test loss:", test_scores['loss']) print("Test accuracy:", test_scores['sparse_categorical_accuracy']) outputs = np.zeros((0, num_classes)) labels = np.array([]) for samples, batch_labels in ds_test: outputs = np.vstack((outputs, model.predict(samples))) labels = np.hstack((labels, batch_labels))
def main(args): """ Main wrapper for training sound event localization and detection network. :param argv: expects two optional inputs. first input: job_id - (optional) all the output files will be uniquely represented with this. (default) 1 second input: task_id - (optional) To chose the system configuration in parameters.py. (default) uses default parameters """ # use parameter set defined by user task_id = args.params params = parameter.get_params(task_id) job_id = args.model_name model_dir = 'models/' + args.author + '/' if args.author != "" else 'models/' utils.create_folder(model_dir) unique_name = '{}_ov{}_split{}_{}{}_3d{}_{}'.format( params['dataset'], params['overlap'], params['split'], params['mode'], params['weakness'], int(params['cnn_3d']), job_id) model_name = unique_name epoch_manager = JSON_Manager(args.author, unique_name) logdir = "logs/" + args.author + "/" + unique_name unique_name = os.path.join(model_dir, unique_name) print("unique_name: {}\n".format(unique_name)) session = tf.InteractiveSession() file_writer = tf.summary.FileWriter(logdir, session.graph) data_gen_train = cls_data_generator.DataGenerator( dataset=params['dataset'], ov=params['overlap'], split=params['split'], db=params['db'], nfft=params['nfft'], batch_size=params['batch_size'], seq_len=params['sequence_length'], classifier_mode=params['mode'], weakness=params['weakness'], datagen_mode='train', cnn3d=params['cnn_3d'], xyz_def_zero=params['xyz_def_zero'], azi_only=params['azi_only']) data_gen_test = cls_data_generator.DataGenerator( dataset=params['dataset'], ov=params['overlap'], split=params['split'], db=params['db'], nfft=params['nfft'], batch_size=params['batch_size'], seq_len=params['sequence_length'], classifier_mode=params['mode'], weakness=params['weakness'], datagen_mode='test', cnn3d=params['cnn_3d'], xyz_def_zero=params['xyz_def_zero'], azi_only=params['azi_only'], shuffle=False) data_in, data_out = data_gen_train.get_data_sizes() print('FEATURES:\n' '\tdata_in: {}\n' '\tdata_out: {}\n'.format(data_in, data_out)) gt = collect_test_labels(data_gen_test, data_out, params['mode'], params['quick_test']) sed_gt = evaluation_metrics.reshape_3Dto2D(gt[0]) doa_gt = evaluation_metrics.reshape_3Dto2D(gt[1]) print('MODEL:\n' '\tdropout_rate: {}\n' '\tCNN: nb_cnn_filt: {}, pool_size{}\n' '\trnn_size: {}, fnn_size: {}\n'.format( params['dropout_rate'], params['nb_cnn3d_filt'] if params['cnn_3d'] else params['nb_cnn2d_filt'], params['pool_size'], params['rnn_size'], params['fnn_size'])) model = keras_model.get_model(data_in=data_in, data_out=data_out, dropout_rate=params['dropout_rate'], nb_cnn2d_filt=params['nb_cnn2d_filt'], pool_size=params['pool_size'], rnn_size=params['rnn_size'], fnn_size=params['fnn_size'], classification_mode=params['mode'], weights=params['loss_weights']) initial = epoch_manager.get_epoch() if initial != 0: print(f"Resume training from epoch {initial}") print("Loading already trained model...") # In order to load custom layers we need to link the references to the custom objects model = load_model(os.path.join(model_dir, model_name + "_model.h5"), custom_objects={ 'QuaternionConv2D': QuaternionConv2D, 'QuaternionGRU': QuaternionGRU, 'QuaternionDense': QuaternionDense }) best_metric = epoch_manager.get_best_metric() best_std = epoch_manager.get_best_std() conf_mat = None best_conf_mat = epoch_manager.get_best_conf_mat() best_epoch = epoch_manager.get_best_epoch() patience_cnt = epoch_manager.get_patience_cnt() epoch_metric_loss = np.zeros(params['nb_epochs']) sed_score = np.zeros(params['nb_epochs']) std_score = np.zeros(params['nb_epochs']) doa_score = np.zeros(params['nb_epochs']) seld_score = np.zeros(params['nb_epochs']) tr_loss = np.zeros(params['nb_epochs']) val_loss = np.zeros(params['nb_epochs']) doa_loss = np.zeros((params['nb_epochs'], 6)) sed_loss = np.zeros((params['nb_epochs'], 2)) time_hold = tf.placeholder(tf.float32, shape=None, name='time_summary') time_summ = tf.summary.scalar('time', time_hold) tr_loss_hold = tf.placeholder(tf.float32, shape=None, name='tr_loss_summary') tr_loss_summ = tf.summary.scalar('tr_loss', tr_loss_hold) val_loss_hold = tf.placeholder(tf.float32, shape=None, name='val_loss_summary') val_loss_summ = tf.summary.scalar('val_loss', val_loss_hold) f1_hold = tf.placeholder(tf.float32, shape=None, name='f1_summary') f1_summ = tf.summary.scalar('F1_overall', f1_hold) er_hold = tf.placeholder(tf.float32, shape=None, name='er_summary') er_summ = tf.summary.scalar('ER_overall', er_hold) doa_error_gt_hold = tf.placeholder(tf.float32, shape=None, name='doa_error_gt_summary') doa_error_gt_summ = tf.summary.scalar('doa_error_gt', doa_error_gt_hold) doa_error_pred_hold = tf.placeholder(tf.float32, shape=None, name='doa_error_pred_summary') doa_error_pred_summ = tf.summary.scalar('doa_error_pred', doa_error_pred_hold) good_pks_hold = tf.placeholder(tf.float32, shape=None, name='good_pks_summary') good_pks_summ = tf.summary.scalar('good_pks_ratio', good_pks_hold) sed_score_hold = tf.placeholder(tf.float32, shape=None, name='sed_score_summary') sed_score_summ = tf.summary.scalar('sed_score', sed_score_hold) doa_score_hold = tf.placeholder(tf.float32, shape=None, name='doa_score_summary') doa_score_summ = tf.summary.scalar('doa_score', doa_score_hold) seld_score_hold = tf.placeholder(tf.float32, shape=None, name='seld_score_summary') seld_score_summ = tf.summary.scalar('seld_score', seld_score_hold) std_score_hold = tf.placeholder(tf.float32, shape=None, name='std_score_summary') std_score_summ = tf.summary.scalar('std_score', std_score_hold) best_error_metric_hold = tf.placeholder(tf.float32, shape=None, name='best_error_metric_summary') best_error_metric_summ = tf.summary.scalar('best_error_metric', best_error_metric_hold) best_epoch_hold = tf.placeholder(tf.float32, shape=None, name='best_epoch_summary') best_epoch_summ = tf.summary.scalar('best_epoch', best_epoch_hold) best_std_hold = tf.placeholder(tf.float32, shape=None, name='best_std_summary') best_std_summ = tf.summary.scalar('best_std', best_std_hold) merged = tf.summary.merge_all() for epoch_cnt in range(initial, params['nb_epochs']): start = time.time() hist = model.fit_generator( generator=data_gen_train.generate(), steps_per_epoch=5 if params['quick_test'] else data_gen_train.get_total_batches_in_data(), validation_data=data_gen_test.generate(), validation_steps=5 if params['quick_test'] else data_gen_test.get_total_batches_in_data(), use_multiprocessing=False, epochs=1, verbose=1) tr_loss[epoch_cnt] = hist.history.get('loss')[-1] val_loss[epoch_cnt] = hist.history.get('val_loss')[-1] pred = model.predict_generator( generator=data_gen_test.generate(), steps=5 if params['quick_test'] else data_gen_test.get_total_batches_in_data(), use_multiprocessing=False, verbose=2) print("pred:", pred[1].shape) if params['mode'] == 'regr': sed_pred = evaluation_metrics.reshape_3Dto2D(pred[0]) > 0.5 doa_pred = evaluation_metrics.reshape_3Dto2D(pred[1]) sed_loss[epoch_cnt, :] = evaluation_metrics.compute_sed_scores( sed_pred, sed_gt, data_gen_test.nb_frames_1s()) if params['azi_only']: doa_loss[ epoch_cnt, :], conf_mat = evaluation_metrics.compute_doa_scores_regr_xy( doa_pred, doa_gt, sed_pred, sed_gt) else: doa_loss[ epoch_cnt, :], conf_mat = evaluation_metrics.compute_doa_scores_regr_xyz( doa_pred, doa_gt, sed_pred, sed_gt) sed_score[epoch_cnt] = np.mean( [sed_loss[epoch_cnt, 0], 1 - sed_loss[epoch_cnt, 1]]) doa_score[epoch_cnt] = np.mean([ 2 * np.arcsin(doa_loss[epoch_cnt, 1] / 2.0) / np.pi, 1 - (doa_loss[epoch_cnt, 5] / float(doa_gt.shape[0])) ]) seld_score[epoch_cnt] = np.mean( [sed_score[epoch_cnt], doa_score[epoch_cnt]]) # standard deviation std_score[epoch_cnt] = np.std( [sed_score[epoch_cnt], doa_score[epoch_cnt]]) plot_functions(unique_name, tr_loss, val_loss, sed_loss, doa_loss, sed_score, doa_score) patience_cnt += 1 epoch_manager.increase_patience_cnt() model.save('{}_model.h5'.format(unique_name)) if seld_score[epoch_cnt] < best_metric: best_metric = seld_score[epoch_cnt] epoch_manager.set_best_metric(best_metric) best_std = std_score[epoch_cnt] epoch_manager.set_best_std(best_std) best_conf_mat = conf_mat epoch_manager.set_best_conf_mat(conf_mat) best_epoch = epoch_cnt epoch_manager.set_best_epoch(best_epoch) model.save('{}_best_model.h5'.format(unique_name)) patience_cnt = 0 epoch_manager.reset_patience_cnt() if patience_cnt > params['patience']: print( f"\n---- PATIENCE TRIGGERED AFTER {epoch_cnt} EPOCHS ----\n") break summary = session.run(merged, feed_dict={ time_hold: time.time() - start, tr_loss_hold: tr_loss[epoch_cnt], val_loss_hold: val_loss[epoch_cnt], f1_hold: sed_loss[epoch_cnt, 1], er_hold: sed_loss[epoch_cnt, 0], doa_error_gt_hold: doa_loss[epoch_cnt, 1], doa_error_pred_hold: doa_loss[epoch_cnt, 2], good_pks_hold: doa_loss[epoch_cnt, 5] / float(sed_gt.shape[0]), sed_score_hold: sed_score[epoch_cnt], doa_score_hold: doa_score[epoch_cnt], seld_score_hold: seld_score[epoch_cnt], std_score_hold: std_score[epoch_cnt], best_error_metric_hold: best_metric, best_epoch_hold: best_epoch, best_std_hold: best_std }) file_writer.add_summary(summary, epoch_cnt) print( 'epoch_cnt: %d, time: %.2fs, tr_loss: %.2f, val_loss: %.2f, ' 'F1_overall: %.2f, ER_overall: %.2f, ' 'doa_error_gt: %.2f, doa_error_pred: %.2f, good_pks_ratio:%.2f, ' 'sed_score: %.2f, doa_score: %.2f, best_error_metric: %.2f, best_epoch : %d, best_std: %.2f' % (epoch_cnt, time.time() - start, tr_loss[epoch_cnt], val_loss[epoch_cnt], sed_loss[epoch_cnt, 1], sed_loss[epoch_cnt, 0], doa_loss[epoch_cnt, 1], doa_loss[epoch_cnt, 2], doa_loss[epoch_cnt, 5] / float(sed_gt.shape[0]), sed_score[epoch_cnt], doa_score[epoch_cnt], best_metric, best_epoch, best_std)) epoch_manager.increase_epoch() lower_confidence, upper_confidence = evaluation_metrics.compute_confidence_interval( best_metric, best_std, params['nb_epochs'], confid_coeff=1.96) # 1.96 for a 95% CI print("\n---- FINISHED TRAINING ----\n") print('best_conf_mat : {}'.format(best_conf_mat)) print('best_conf_mat_diag : {}'.format(np.diag(best_conf_mat))) print('saved model for the best_epoch: {} with best_metric: {}, '.format( best_epoch, best_metric)) print( 'DOA Metrics: doa_loss_gt: {}, doa_loss_pred: {}, good_pks_ratio: {}'. format(doa_loss[best_epoch, 1], doa_loss[best_epoch, 2], doa_loss[best_epoch, 5] / float(sed_gt.shape[0]))) print('SED Metrics: ER_overall: {}, F1_overall: {}'.format( sed_loss[best_epoch, 0], sed_loss[best_epoch, 1])) print('Confidence Interval: lower_interval: {}, upper_inteval: {}'.format( lower_confidence, upper_confidence)) print('unique_name: {} '.format(unique_name))
# set path machine_type = os.path.split(target_dir)[1] model_file_path = "{model}/model_{machine_type}.hdf5".format(model=param["model_directory"], machine_type=machine_type) history_img = "{model}/history_{machine_type}.png".format(model=param["model_directory"], machine_type=machine_type) if os.path.exists(model_file_path): com.logger.info("model exists") continue # train model print("============== MODEL CREATING ==============") model = keras_model.get_model(param['feature']['n_mels'] * param["feature"]["frames"]) model.summary() model.compile(**param["fit"]["compile"]) # generate dataset for method_augm in param['feature']['augmentation']: print("============== DATASET_", method_augm, "_GENERATOR ==============") files = file_list_generator(target_dir)[:3000] train_data = list_to_vector_array(files, msg="generate train_dataset", n_mels=param["feature"]["n_mels"], frames=param["feature"]["frames"], n_fft=param["feature"]["n_fft"], hop_length=param["feature"]["hop_length"], power=param["feature"]["power"],
train_data = list_to_vector_array( files, False, msg="generate train_dataset", n_mels=param["feature"]["n_mels"], frames=param["feature"]["frames"], n_fft=param["feature"]["n_fft"], hop_length=param["feature"]["hop_length"], power=param["feature"]["power"]) print("============== MODEL TRAINING ==============") ## Load pre-trained model # model = keras_model.get_model(param["feature"]["n_mels"] * param["feature"]["frames"]) # model = keras.models.load_model("../dcase2020_task2_baseline/{model}/model_{machine_type}.hdf5".format(model=param["model_directory"], machine_type=machine_type)) model = keras_model.get_model(100) # model.summary() model.compile(**param["fit"]["compile"]) # model.compile(**param["fit"]["compile"]) history = model.fit(train_data, train_data, epochs=100, batch_size=param["fit"]["batch_size"], shuffle=param["fit"]["shuffle"], validation_split=param["fit"]["validation_split"], verbose=param["fit"]["verbose"]) visualizer.loss_plot(history.history["loss"], history.history["val_loss"]) visualizer.save_figure(history_img)
def main(argv): """ Main wrapper for training sound event localization and detection network. :param argv: expects two optional inputs. first input: job_id - (optional) all the output files will be uniquely represented with this. (default) 1 second input: task_id - (optional) To chose the system configuration in parameters.py. (default) uses default parameters """ if len(argv) != 3: print('\n\n') print( '-------------------------------------------------------------------------------------------------------' ) print('The code expected two inputs') print('\t>> python seld.py <job-id> <task-id>') print( '\t\t<job-id> is a unique identifier which is used for output filenames (models, training plots). ' 'You can use any number or string for this.') print( '\t\t<task-id> is used to choose the user-defined parameter set from parameter.py' ) print('Using default inputs for now') print( '-------------------------------------------------------------------------------------------------------' ) print('\n\n') # use parameter set defined by user task_id = '1' if len(argv) < 3 else argv[-1] params = parameter.get_params(task_id) job_id = 1 if len(argv) < 2 else argv[1] model_dir = 'models/' utils.create_folder(model_dir) unique_name = '{}_ov{}_split{}_{}{}_3d{}_{}'.format( params['dataset'], params['overlap'], params['split'], params['mode'], params['weakness'], int(params['cnn_3d']), job_id) unique_name = os.path.join(model_dir, unique_name) print("unique_name: {}\n".format(unique_name)) data_gen_train = cls_data_generator.DataGenerator( dataset=params['dataset'], ov=params['overlap'], split=params['split'], db=params['db'], nfft=params['nfft'], batch_size=params['batch_size'], seq_len=params['sequence_length'], classifier_mode=params['mode'], weakness=params['weakness'], datagen_mode='train', cnn3d=params['cnn_3d'], xyz_def_zero=params['xyz_def_zero'], azi_only=params['azi_only']) data_gen_test = cls_data_generator.DataGenerator( dataset=params['dataset'], ov=params['overlap'], split=params['split'], db=params['db'], nfft=params['nfft'], batch_size=params['batch_size'], seq_len=params['sequence_length'], classifier_mode=params['mode'], weakness=params['weakness'], datagen_mode='test', cnn3d=params['cnn_3d'], xyz_def_zero=params['xyz_def_zero'], azi_only=params['azi_only'], shuffle=False) data_in, data_out = data_gen_train.get_data_sizes() print('FEATURES:\n' '\tdata_in: {}\n' '\tdata_out: {}\n'.format(data_in, data_out)) gt = collect_test_labels(data_gen_test, data_out, params['mode'], params['quick_test']) sed_gt = evaluation_metrics.reshape_3Dto2D(gt[0]) doa_gt = evaluation_metrics.reshape_3Dto2D(gt[1]) print('MODEL:\n' '\tdropout_rate: {}\n' '\tCNN: nb_cnn_filt: {}, pool_size{}\n' '\trnn_size: {}, fnn_size: {}\n'.format( params['dropout_rate'], params['nb_cnn3d_filt'] if params['cnn_3d'] else params['nb_cnn2d_filt'], params['pool_size'], params['rnn_size'], params['fnn_size'])) # TPU CODE FOR GOOGLE COLABORATORY resolver = tf.distribute.cluster_resolver.TPUClusterResolver( tpu='grpc://' + os.environ['COLAB_TPU_ADDR']) tf.config.experimental_connect_to_cluster(resolver) # This is the TPU initialization code that has to be at the beginning. tf.tpu.experimental.initialize_tpu_system(resolver) print("All devices: ", tf.config.list_logical_devices('TPU')) strategy = tf.distribute.experimental.TPUStrategy(resolver) with strategy.scope(): # Load or create model model = utils.load_model(unique_name) if model is None: model = keras_model.get_model( data_in=data_in, data_out=data_out, dropout_rate=params['dropout_rate'], nb_cnn2d_filt=params['nb_cnn2d_filt'], pool_size=params['pool_size'], rnn_size=params['rnn_size'], fnn_size=params['fnn_size'], classification_mode=params['mode'], weights=params['loss_weights']) model.summary() best_metric = 99999 conf_mat = None best_conf_mat = None best_epoch = -1 patience_cnt = 0 epoch_metric_loss = np.zeros(params['nb_epochs']) sed_score = np.zeros(params['nb_epochs']) doa_score = np.zeros(params['nb_epochs']) tr_loss = np.zeros(params['nb_epochs']) val_loss = np.zeros(params['nb_epochs']) doa_loss = np.zeros((params['nb_epochs'], 6)) sed_loss = np.zeros((params['nb_epochs'], 2)) for epoch_cnt in range(params['nb_epochs']): start = time.time() hist = model.fit_generator( generator=data_gen_train.generate(), steps_per_epoch=5 if params['quick_test'] else data_gen_train.get_total_batches_in_data(), validation_data=data_gen_test.generate(), validation_steps=5 if params['quick_test'] else data_gen_test.get_total_batches_in_data(), use_multiprocessing=False, epochs=1, verbose=1) tr_loss[epoch_cnt] = hist.history.get('loss')[-1] val_loss[epoch_cnt] = hist.history.get('val_loss')[-1] pred = model.predict_generator( generator=data_gen_test.generate(), steps=5 if params['quick_test'] else data_gen_test.get_total_batches_in_data(), use_multiprocessing=False, verbose=2) print("pred:", pred[1].shape) if params['mode'] == 'regr': sed_pred = evaluation_metrics.reshape_3Dto2D(pred[0]) > 0.5 doa_pred = evaluation_metrics.reshape_3Dto2D(pred[1]) sed_loss[epoch_cnt, :] = evaluation_metrics.compute_sed_scores( sed_pred, sed_gt, data_gen_test.nb_frames_1s()) if params['azi_only']: doa_loss[ epoch_cnt, :], conf_mat = evaluation_metrics.compute_doa_scores_regr_xy( doa_pred, doa_gt, sed_pred, sed_gt) else: doa_loss[ epoch_cnt, :], conf_mat = evaluation_metrics.compute_doa_scores_regr_xyz( doa_pred, doa_gt, sed_pred, sed_gt) # epoch_metric_loss[epoch_cnt] = np.mean([ # sed_loss[epoch_cnt, 0], # 1-sed_loss[epoch_cnt, 1], # 2*np.arcsin(doa_loss[epoch_cnt, 1]/2.0)/np.pi, # 1 - (doa_loss[epoch_cnt, 5] / float(doa_gt.shape[0]))] # ) sed_score[epoch_cnt] = np.mean( [sed_loss[epoch_cnt, 0], 1 - sed_loss[epoch_cnt, 1]]) doa_score[epoch_cnt] = np.mean([ 2 * np.arcsin(doa_loss[epoch_cnt, 1] / 2.0) / np.pi, 1 - (doa_loss[epoch_cnt, 5] / float(doa_gt.shape[0])) ]) #plot_functions(unique_name, tr_loss, val_loss, sed_loss, doa_loss, epoch_metric_loss) plot_functions(unique_name, tr_loss, val_loss, sed_loss, doa_loss, sed_score, doa_score) patience_cnt += 1 # if epoch_metric_loss[epoch_cnt] < best_metric: # best_metric = epoch_metric_loss[epoch_cnt] # best_conf_mat = conf_mat # best_epoch = epoch_cnt # model.save('{}_model.h5'.format(unique_name)) # patience_cnt = 0 if sed_score[epoch_cnt] < best_metric: best_metric = sed_score[epoch_cnt] best_conf_mat = conf_mat best_epoch = epoch_cnt model.save('{}_model.h5'.format(unique_name)) patience_cnt = 0 print( 'epoch_cnt: %d, time: %.2fs, tr_loss: %.2f, val_loss: %.2f, ' 'F1_overall: %.2f, ER_overall: %.2f, ' 'doa_error_gt: %.2f, doa_error_pred: %.2f, good_pks_ratio:%.2f, ' 'sed_score: %.2f, doa_score: %.2f, best_error_metric: %.2f, best_epoch : %d' % (epoch_cnt, time.time() - start, tr_loss[epoch_cnt], val_loss[epoch_cnt], sed_loss[epoch_cnt, 1], sed_loss[epoch_cnt, 0], doa_loss[epoch_cnt, 1], doa_loss[epoch_cnt, 2], doa_loss[epoch_cnt, 5] / float(sed_gt.shape[0]), sed_score[epoch_cnt], doa_score[epoch_cnt], best_metric, best_epoch)) #plot_functions(unique_name, tr_loss, val_loss, sed_loss, doa_loss, sed_score, doa_score, epoch_cnt) print('best_conf_mat : {}'.format(best_conf_mat)) print('best_conf_mat_diag : {}'.format(np.diag(best_conf_mat))) print('saved model for the best_epoch: {} with best_metric: {}, '.format( best_epoch, best_metric)) print( 'DOA Metrics: doa_loss_gt: {}, doa_loss_pred: {}, good_pks_ratio: {}'. format(doa_loss[best_epoch, 1], doa_loss[best_epoch, 2], doa_loss[best_epoch, 5] / float(sed_gt.shape[0]))) print('SED Metrics: ER_overall: {}, F1_overall: {}'.format( sed_loss[best_epoch, 0], sed_loss[best_epoch, 1])) print('unique_name: {} '.format(unique_name))
def main(params, network_params): print( '\n\n----------------------------------------------------------------------------------------------------' ) print( '----------------------------------- Splitting data into {:0>2d} folds -----------------------------------' .format(params['nb_folds'])) print( '----------------------------------------------------------------------------------------------------' ) evaluation_cls = evaluation_metrics(results_dir=params['results_dir'], win_len=params['win_len'], hop_len=params['hop_len'], seq_len=params['seq_len'], seq_hop_len=params['seq_hop_len'], nb_channels=params['nb_channels'], nb_classes=params['nb_classes'], silent=True) data_splitter_cls = data_splitter(data_dir=params['data_dir'], win_len=params['win_len'], hop_len=params['hop_len'], seq_len=params['seq_len'], seq_hop_len=params['seq_hop_len'], nb_channels=params['nb_channels'], nb_classes=params['nb_classes'], validate=False, nb_folds=params['nb_folds'], silent=True) data_splitter_cls.perform() Path(os.path.join(params['log_dir'])).mkdir(parents=True, exist_ok=True) Path(os.path.join(params['results_dir'])).mkdir(parents=True, exist_ok=True) for fold_cnt in range(params['nb_folds']): print( '\n\n----------------------------------------------------------------------------------------------------' ) print( '------------------------------------------ fold: {:0>2d} ------------------------------------------' .format(fold_cnt + 1)) print( '----------------------------------------------------------------------------------------------------' ) model_name = os.path.join(params['log_dir'], 'fold_{}_best_model.h5'.format(fold_cnt + 1)) csv_logger = CSVLogger(filename=os.path.join( params['log_dir'], 'fold_{}_log.csv'.format(fold_cnt + 1))) early_stopper = EarlyStopping(monitor='loss', min_delta=0, mode='min') data_splitter_cls.prepare(fold_n=fold_cnt, run_type='train') data_splitter_cls.prepare(fold_n=fold_cnt, run_type='val') data_splitter_cls.prepare(fold_n=fold_cnt, run_type='test') print('Loading training dataset:') train_data_gen = DataGenerator(data_dir=params['data_dir'], shuffle=False, win_len=params['win_len'], hop_len=params['hop_len'], seq_len=params['seq_len'], seq_hop_len=params['seq_hop_len'], nb_channels=params['nb_channels'], nb_classes=params['nb_classes'], n_fold=fold_cnt, batch_size=params['batch_size'], run_type='train') val_data_gen = DataGenerator(data_dir=params['data_dir'], shuffle=False, win_len=params['win_len'], hop_len=params['hop_len'], seq_len=params['seq_len'], seq_hop_len=params['seq_hop_len'], nb_channels=params['nb_channels'], nb_classes=params['nb_classes'], n_fold=fold_cnt, batch_size=params['batch_size'], run_type='val') test_data_gen = DataGenerator(data_dir=params['data_dir'], shuffle=False, win_len=params['win_len'], hop_len=params['hop_len'], seq_len=params['seq_len'], seq_hop_len=params['seq_hop_len'], nb_channels=params['nb_channels'], nb_classes=params['nb_classes'], n_fold=fold_cnt, batch_size=params['batch_size'], run_type='test') model = get_model(params, network_params) #for epoch_cnt in range(params['nb_epochs']): #print('Epoch No. {}'.format(epoch_cnt+1)) hist = model.fit(x=train_data_gen, validation_data=val_data_gen, epochs=params['nb_epochs'], workers=10, use_multiprocessing=True, callbacks=[csv_logger]) #early_stopper,csv_logger]) model.save(model_name) print( '\nLoading the best model and predicting results on the testing data' ) model = load_model(model_name) pred_test = model.predict(x=test_data_gen, workers=2, use_multiprocessing=False) pred_test_logits = pred_test pred_test = pred_test > 0.5 test_data_records = pd.read_csv( os.path.join( test_data_gen._data_dir, 'folds_metadata', '{}_metadata_{}.csv'.format(test_data_gen._run_type, fold_cnt + 1))) test_h5 = h5py.File( os.path.join(test_data_gen._data_dir, 'folds_h5', '{}_metadata_{}.hdf5'.format('test', fold_cnt + 1))) ref_test = np.asarray(test_h5['seq_labels']) test_h5.close() Path( os.path.join(params['results_dir'], 'test_val_fold{:0>2d}'.format(fold_cnt + 1))).mkdir( parents=True, exist_ok=True) Path( os.path.join( params['results_dir'], 'test_val_logits_fold{:0>2d}'.format(fold_cnt + 1))).mkdir( parents=True, exist_ok=True) Path( os.path.join(params['results_dir'], 'ref_val_fold{:0>2d}'.format(fold_cnt + 1))).mkdir( parents=True, exist_ok=True) evaluation_cls.combine_seqlogits(pred_test, test_data_records, run_type='test', fold_n=fold_cnt) evaluation_cls.combine_seqlogits(ref_test, test_data_records, run_type='ref', fold_n=fold_cnt) evaluation_cls.combine_seqlogitspro(pred_test_logits, test_data_records, fold_n=fold_cnt) nb_test_files = os.listdir( os.path.join(params['results_dir'], 'test_val_fold{:0>2d}'.format(fold_cnt + 1))) acc = np.zeros((len(nb_test_files), 1)) sens = np.zeros((len(nb_test_files), 1)) spec = np.zeros((len(nb_test_files), 1)) fold_clogits = [] fold_cgtruth = [] print('Model evaluation for fold#{}:'.format(fold_cnt + 1)) for testfile_cnt, testfile in enumerate(nb_test_files): print('Evaluating segmentation of file# {}: {}'.format( testfile_cnt + 1, testfile)) acc[testfile_cnt], sens[testfile_cnt], spec[ testfile_cnt] = evaluation_cls.evaluate_sequences( np.load( os.path.join( params['results_dir'], 'ref_val_fold{:0>2d}'.format(fold_cnt + 1), testfile)), np.load( os.path.join( params['results_dir'], 'test_val_fold{:0>2d}'.format(fold_cnt + 1), testfile))) fold_clogits = np.concatenate( (fold_clogits, np.load( os.path.join( params['results_dir'], 'test_val_logits_fold{:0>2d}'.format(fold_cnt + 1), testfile)))) fold_cgtruth = np.concatenate( (fold_cgtruth, np.load( os.path.join(params['results_dir'], 'ref_val_fold{:0>2d}'.format(fold_cnt + 1), testfile)))) np.save( os.path.join( params['results_dir'], 'fold_{:0>2d}_aggregate_logits.npy'.format(fold_cnt + 1)), fold_clogits) np.save( os.path.join( params['results_dir'], 'fold_{:0>2d}_aggregate_gtruth.npy'.format(fold_cnt + 1)), fold_cgtruth) np.save( os.path.join(params['results_dir'], 'fold_{:0>2d}_qmeasures_acc.npy'.format(fold_cnt + 1)), acc) np.save( os.path.join( params['results_dir'], 'fold_{:0>2d}_qmeasures_sens.npy'.format(fold_cnt + 1)), sens) np.save( os.path.join( params['results_dir'], 'fold_{:0>2d}_qmeasures_spec.npy'.format(fold_cnt + 1)), spec)
def main(argv): """ Main wrapper for training sound event localization and detection network. :param argv: expects two optional inputs. first input: job_id - (optional) all the output files will be uniquely represented with this. (default) 1 second input: task_id - (optional) To chose the system configuration in parameters.py. (default) uses default parameters """ if len(argv) != 3: print('\n\n') print('-------------------------------------------------------------------------------------------------------') print('The code expected two inputs') print('\t>> python seld.py <job-id> <task-id>') print('\t\t<job-id> is a unique identifier which is used for output filenames (models, training plots). ' 'You can use any number or string for this.') print('\t\t<task-id> is used to choose the user-defined parameter set from parameter.py') print('Using default inputs for now') print('-------------------------------------------------------------------------------------------------------') print('\n\n') # use parameter set defined by user task_id = '1' if len(argv) < 3 else argv[-1] params = parameter.get_params(task_id) job_id = 1 if len(argv) < 2 else argv[1] model_dir = 'models/' utils.create_folder(model_dir) unique_name = '{}_train{}_validation{}_seq{}'.format(params['dataset'], params['train_split'], params['val_split'], params['sequence_length']) unique_name = os.path.join(model_dir, unique_name) print("unique_name: {}\n".format(unique_name)) # Cycling over overlaps for ov in range(1, params['overlap']+1): data_gen_test = cls_data_generator.DataGenerator( dataset=params['dataset'], ov=params['overlap'], ov_num=ov, split=params['test_split'], db=params['db'], nfft=params['nfft'], batch_size=params['batch_size'], seq_len=params['sequence_length'], classifier_mode=params['mode'], weakness=params['weakness'], datagen_mode='test', cnn3d=params['cnn_3d'], xyz_def_zero=params['xyz_def_zero'], azi_only=params['azi_only'], shuffle=False ) data_in, data_out = data_gen_test.get_data_sizes() n_classes = data_out[0][2] print( 'FEATURES:\n' '\tdata_in: {}\n' '\tdata_out: {}\n'.format( data_in, data_out ) ) gt = collect_test_labels(data_gen_test, data_out, params['mode'], params['quick_test']) sed_gt = evaluation_metrics.reshape_3Dto2D(gt[0]) doa_gt = evaluation_metrics.reshape_3Dto2D(gt[1]) print("#### Saving DOA and SED GT Values ####") f = open("models/doa_gt.txt", "w+") for elem in doa_gt: f.write(str(list(elem)) + "\n") f.close() f = open("models/sed_gt.txt", "w+") for elem in sed_gt: f.write(str(elem)+"\n") f.close() print("######################################") print( 'MODEL:\n' '\tdropout_rate: {}\n' '\tCNN: nb_cnn_filt: {}, pool_size{}\n' '\trnn_size: {}, fnn_size: {}\n'.format( params['dropout_rate'], params['nb_cnn3d_filt'] if params['cnn_3d'] else params['nb_cnn2d_filt'], params['pool_size'], params['rnn_size'], params['fnn_size'] ) ) model = keras_model.get_model(data_in=data_in, data_out=data_out, dropout_rate=params['dropout_rate'], nb_cnn2d_filt=params['nb_cnn2d_filt'], pool_size=params['pool_size'], rnn_size=params['rnn_size'], fnn_size=params['fnn_size'], classification_mode=params['mode'], weights=params['loss_weights'], summary=False) if(os.path.exists('{}_model.ckpt'.format(unique_name))): print("Model found!") model.load_weights('{}_model.ckpt'.format(unique_name)) for i in range(10): print("###") sed_score = np.zeros(params['nb_epochs']) doa_score = np.zeros(params['nb_epochs']) seld_score = np.zeros(params['nb_epochs']) tr_loss = np.zeros(params['nb_epochs']) val_loss = np.zeros(params['nb_epochs']) doa_loss = np.zeros((params['nb_epochs'], 6)) sed_loss = np.zeros((params['nb_epochs'], 2)) epoch_cnt = 0 start = time.time() print("#### Prediction on validation split ####") pred = model.predict_generator( generator=data_gen_test.generate(), steps=params['quick_test_steps'] if params['quick_test'] else data_gen_test.get_total_batches_in_data(), use_multiprocessing=False, workers=1, verbose=1, ) print("##########################") #print("pred:", pred[1].shape) if params['mode'] == 'regr': sed_pred = np.array(evaluation_metrics.reshape_3Dto2D(pred[0])) > .5 doa_pred = evaluation_metrics.reshape_3Dto2D(pred[1]) print("#### Saving DOA and SED Pred Values ####") f = open("models/doa_pred.txt", "w+") for elem in doa_pred: f.write(str(list(elem)) + "\n") f.close() f = open("models/sed_pred.txt", "w+") for elem in sed_pred: f.write(str(elem)+"\n") f.close() print("########################################") # Old version of confidence intervals ''' # Computing confidence intervals sed_err = sed_gt - sed_pred [sed_conf_low, sed_conf_up, sed_median] = compute_confidence(sed_err) # print("Condidence Interval for SED error is [" + str(sed_conf_low) + ", " + str(sed_conf_up) + "]") print("Confidence Interval for SED error is [ %f, %f ]" % (sed_conf_low, sed_conf_up)) # print("\tMedian is " + str(sed_median)) print("\tMedian is %f" % (sed_median)) # print("\tDisplacement: +/- " + str(sed_conf_up - sed_median)) print("\tDisplacement: +/- %f" % (sed_conf_up - sed_median)) doa_err = doa_gt - doa_pred [doa_conf_low, doa_conf_up, doa_median] = compute_confidence(doa_err) # print("Condidence Interval for DOA is [" + str(doa_conf_low) + ", " + str(doa_conf_up) + "]") print("Confidence Interval for DOA is [ %f, %f ]" % (doa_conf_low, doa_conf_up)) # print("Median is " + str(doa_median)) print("\tMedian is %f" % (doa_median)) # print("Displacement: +/- " + str(doa_conf_up - doa_median)) print("\tDisplacement: +/- %f" % (doa_conf_up - doa_median)) # ------------------------------ ''' sed_loss[epoch_cnt, :] = evaluation_metrics.compute_sed_scores(sed_pred, sed_gt, data_gen_test.nb_frames_1s()) if params['azi_only']: doa_loss[epoch_cnt, :], conf_mat = evaluation_metrics.compute_doa_scores_regr_xy(doa_pred, doa_gt, sed_pred, sed_gt) else: doa_loss[epoch_cnt, :], conf_mat = evaluation_metrics.compute_doa_scores_regr_xyz(doa_pred, doa_gt, sed_pred, sed_gt) sed_score[epoch_cnt] = np.mean([sed_loss[epoch_cnt, 0], 1 - sed_loss[epoch_cnt, 1]]) doa_score[epoch_cnt] = np.mean([2 * np.arcsin(doa_loss[epoch_cnt, 1] / 2.0) / np.pi, 1 - (doa_loss[epoch_cnt, 5] / float(doa_gt.shape[0]))]) seld_score[epoch_cnt] = (sed_score[epoch_cnt] + doa_score[epoch_cnt]) / 2 if os.path.isdir('./models'): plot.imshow(conf_mat, cmap='binary', interpolation='None') plot.savefig('models/confusion_matrix.jpg') # New confidence computation, differing doa and sed errors sed_err = sed_loss[epoch_cnt, 0] [sed_conf_low, sed_conf_up] = compute_confidence(sed_err, sed_pred.shape[0]) print("Confidence Interval for SED error is [ %f, %f ]" % (sed_conf_low, sed_conf_up)) doa_err = doa_gt - doa_pred [x_err, y_err, z_err] = compute_doa_confidence(doa_err, n_classes) print('epoch_cnt: %d, time: %.2fs, tr_loss: %.4f, val_loss: %.4f, ' 'F1_overall: %.2f, ER_overall: %.2f, ' 'doa_error_gt: %.2f, doa_error_pred: %.2f, good_pks_ratio:%.2f, ' 'sed_score: %.4f, doa_score: %.4f, seld_score: %.4f' % ( epoch_cnt, time.time() - start, tr_loss[epoch_cnt], val_loss[epoch_cnt], sed_loss[epoch_cnt, 1], sed_loss[epoch_cnt, 0], doa_loss[epoch_cnt, 1], doa_loss[epoch_cnt, 2], doa_loss[epoch_cnt, 5] / float(sed_gt.shape[0]), sed_score[epoch_cnt], doa_score[epoch_cnt], seld_score[epoch_cnt] ) ) simple_plotter.plot_3d("models/doa_gt.txt", "models/doa_pred.txt", 0, 11, 200) simple_plotter.plot_confidence(x_err, y_err, z_err, "ov"+str(ov))
def main(argv): """ Main wrapper for training sound event localization and detection network. :param argv: expects two optional inputs. first input: task_id - (optional) To chose the system configuration in parameters.py. (default) 1 - uses default parameters second input: job_id - (optional) all the output files will be uniquely represented with this. (default) 1 """ if len(argv) != 3: print('\n\n') print( '-------------------------------------------------------------------------------------------------------' ) print('The code expected two optional inputs') print('\t>> python seld.py <task-id> <job-id>') print( '\t\t<task-id> is used to choose the user-defined parameter set from parameter.py' ) print('Using default inputs for now') print( '\t\t<job-id> is a unique identifier which is used for output filenames (models, training plots). ' 'You can use any number or string for this.') print( '-------------------------------------------------------------------------------------------------------' ) print('\n\n') # use parameter set defined by user task_id = '1' if len(argv) < 2 else argv[1] params = parameter.get_params(task_id) job_id = 1 if len(argv) < 3 else argv[-1] train_splits, val_splits, test_splits = None, None, None if params['mode'] == 'dev': test_splits = [1, 2, 3, 4] val_splits = [2, 3, 4, 1] train_splits = [[3, 4], [4, 1], [1, 2], [2, 3]] # SUGGESTION: Considering the long training time, major tuning of the method can be done on the first split. # Once you finlaize the method you can evaluate its performance on the complete cross-validation splits # test_splits = [1] # val_splits = [2] # train_splits = [[3, 4]] elif params['mode'] == 'eval': test_splits = [0] val_splits = [1] train_splits = [[2, 3, 4]] avg_scores_val = [] avg_scores_test = [] for split_cnt, split in enumerate(test_splits): print( '\n\n---------------------------------------------------------------------------------------------------' ) print( '------------------------------------ SPLIT {} -----------------------------------------------' .format(split)) print( '---------------------------------------------------------------------------------------------------' ) # Unique name for the run cls_feature_class.create_folder(params['model_dir']) unique_name = '{}_{}_{}_{}_split{}'.format(task_id, job_id, params['dataset'], params['mode'], split) unique_name = os.path.join(params['model_dir'], unique_name) model_name = '{}_model.h5'.format(unique_name) print("unique_name: {}\n".format(unique_name)) # Load train and validation data print('Loading training dataset:') data_gen_train = cls_data_generator.DataGenerator( dataset=params['dataset'], split=train_splits[split_cnt], batch_size=params['batch_size'], seq_len=params['sequence_length'], feat_label_dir=params['feat_label_dir']) print('Loading validation dataset:') data_gen_val = cls_data_generator.DataGenerator( dataset=params['dataset'], split=val_splits[split_cnt], batch_size=params['batch_size'], seq_len=params['sequence_length'], feat_label_dir=params['feat_label_dir'], shuffle=False) # Collect the reference labels for validation data data_in, data_out = data_gen_train.get_data_sizes() print('FEATURES:\n\tdata_in: {}\n\tdata_out: {}\n'.format( data_in, data_out)) gt = collect_test_labels(data_gen_val, data_out, params['quick_test']) sed_gt = evaluation_metrics.reshape_3Dto2D(gt[0]) doa_gt = evaluation_metrics.reshape_3Dto2D(gt[1]) # rescaling the reference elevation data from [-180 180] to [-def_elevation def_elevation] for scoring purpose nb_classes = data_gen_train.get_nb_classes() def_elevation = data_gen_train.get_default_elevation() doa_gt[:, nb_classes:] = doa_gt[:, nb_classes:] / (180. / def_elevation) print( 'MODEL:\n\tdropout_rate: {}\n\tCNN: nb_cnn_filt: {}, pool_size{}\n\trnn_size: {}, fnn_size: {}\n' .format(params['dropout_rate'], params['nb_cnn2d_filt'], params['pool_size'], params['rnn_size'], params['fnn_size'])) model = keras_model.get_model(data_in=data_in, data_out=data_out, dropout_rate=params['dropout_rate'], nb_cnn2d_filt=params['nb_cnn2d_filt'], pool_size=params['pool_size'], rnn_size=params['rnn_size'], fnn_size=params['fnn_size'], weights=params['loss_weights']) best_seld_metric = 99999 best_epoch = -1 patience_cnt = 0 seld_metric = np.zeros(params['nb_epochs']) tr_loss = np.zeros(params['nb_epochs']) val_loss = np.zeros(params['nb_epochs']) doa_metric = np.zeros((params['nb_epochs'], 6)) sed_metric = np.zeros((params['nb_epochs'], 2)) nb_epoch = 2 if params['quick_test'] else params['nb_epochs'] # start training for epoch_cnt in range(nb_epoch): start = time.time() # train once per epoch hist = model.fit_generator( generator=data_gen_train.generate(), steps_per_epoch=2 if params['quick_test'] else data_gen_train.get_total_batches_in_data(), validation_data=data_gen_val.generate(), validation_steps=2 if params['quick_test'] else data_gen_val.get_total_batches_in_data(), epochs=params['epochs_per_fit'], verbose=2) tr_loss[epoch_cnt] = hist.history.get('loss')[-1] val_loss[epoch_cnt] = hist.history.get('val_loss')[-1] # predict once per peoch pred = model.predict_generator( generator=data_gen_val.generate(), steps=2 if params['quick_test'] else data_gen_val.get_total_batches_in_data(), verbose=2) # Calculate the metrics sed_pred = evaluation_metrics.reshape_3Dto2D(pred[0]) > 0.5 doa_pred = evaluation_metrics.reshape_3Dto2D(pred[1]) # rescaling the elevation data from [-180 180] to [-def_elevation def_elevation] for scoring purpose doa_pred[:, nb_classes:] = doa_pred[:, nb_classes:] / (180. / def_elevation) sed_metric[epoch_cnt, :] = evaluation_metrics.compute_sed_scores( sed_pred, sed_gt, data_gen_val.nb_frames_1s()) doa_metric[ epoch_cnt, :] = evaluation_metrics.compute_doa_scores_regr( doa_pred, doa_gt, sed_pred, sed_gt) seld_metric[epoch_cnt] = evaluation_metrics.compute_seld_metric( sed_metric[epoch_cnt, :], doa_metric[epoch_cnt, :]) # Visualize the metrics with respect to epochs plot_functions(unique_name, tr_loss, val_loss, sed_metric, doa_metric, seld_metric) patience_cnt += 1 if seld_metric[epoch_cnt] < best_seld_metric: best_seld_metric = seld_metric[epoch_cnt] best_epoch = epoch_cnt model.save(model_name) patience_cnt = 0 print( 'epoch_cnt: %d, time: %.2fs, tr_loss: %.2f, val_loss: %.2f, ' 'ER_overall: %.2f, F1_overall: %.2f, ' 'doa_error_pred: %.2f, good_pks_ratio:%.2f, ' 'seld_score: %.2f, best_seld_score: %.2f, best_epoch : %d\n' % (epoch_cnt, time.time() - start, tr_loss[epoch_cnt], val_loss[epoch_cnt], sed_metric[epoch_cnt, 0], sed_metric[epoch_cnt, 1], doa_metric[epoch_cnt, 0], doa_metric[epoch_cnt, 1], seld_metric[epoch_cnt], best_seld_metric, best_epoch)) if patience_cnt > params['patience']: break avg_scores_val.append([ sed_metric[best_epoch, 0], sed_metric[best_epoch, 1], doa_metric[best_epoch, 0], doa_metric[best_epoch, 1], best_seld_metric ]) print('\nResults on validation split:') print('\tUnique_name: {} '.format(unique_name)) print('\tSaved model for the best_epoch: {}'.format(best_epoch)) print('\tSELD_score: {}'.format(best_seld_metric)) print('\tDOA Metrics: DOA_error: {}, frame_recall: {}'.format( doa_metric[best_epoch, 0], doa_metric[best_epoch, 1])) print('\tSED Metrics: ER_overall: {}, F1_overall: {}\n'.format( sed_metric[best_epoch, 0], sed_metric[best_epoch, 1])) # ------------------ Calculate metric scores for unseen test split --------------------------------- print('Loading testing dataset:') data_gen_test = cls_data_generator.DataGenerator( dataset=params['dataset'], split=split, batch_size=params['batch_size'], seq_len=params['sequence_length'], feat_label_dir=params['feat_label_dir'], shuffle=False, per_file=params['dcase_output'], is_eval=True if params['mode'] is 'eval' else False) print( '\nLoading the best model and predicting results on the testing split' ) model = load_model('{}_model.h5'.format(unique_name)) pred_test = model.predict_generator( generator=data_gen_test.generate(), steps=2 if params['quick_test'] else data_gen_test.get_total_batches_in_data(), verbose=2) test_sed_pred = evaluation_metrics.reshape_3Dto2D(pred_test[0]) > 0.5 test_doa_pred = evaluation_metrics.reshape_3Dto2D(pred_test[1]) # rescaling the elevation data from [-180 180] to [-def_elevation def_elevation] for scoring purpose test_doa_pred[:, nb_classes:] = test_doa_pred[:, nb_classes:] / ( 180. / def_elevation) if params['dcase_output']: # Dump results in DCASE output format for calculating final scores dcase_dump_folder = os.path.join( params['dcase_dir'], '{}_{}_{}'.format(task_id, params['dataset'], params['mode'])) cls_feature_class.create_folder(dcase_dump_folder) print('Dumping recording-wise results in: {}'.format( dcase_dump_folder)) test_filelist = data_gen_test.get_filelist() # Number of frames for a 60 second audio with 20ms hop length = 3000 frames max_frames_with_content = data_gen_test.get_nb_frames() # Number of frames in one batch (batch_size* sequence_length) consists of all the 3000 frames above with # zero padding in the remaining frames frames_per_file = data_gen_test.get_frame_per_file() for file_cnt in range(test_sed_pred.shape[0] // frames_per_file): output_file = os.path.join( dcase_dump_folder, test_filelist[file_cnt].replace('.npy', '.csv')) dc = file_cnt * frames_per_file output_dict = evaluation_metrics.regression_label_format_to_output_format( data_gen_test, test_sed_pred[dc:dc + max_frames_with_content, :], test_doa_pred[dc:dc + max_frames_with_content, :] * 180 / np.pi) evaluation_metrics.write_output_format_file( output_file, output_dict) if params['mode'] is 'dev': test_data_in, test_data_out = data_gen_test.get_data_sizes() test_gt = collect_test_labels(data_gen_test, test_data_out, params['quick_test']) test_sed_gt = evaluation_metrics.reshape_3Dto2D(test_gt[0]) test_doa_gt = evaluation_metrics.reshape_3Dto2D(test_gt[1]) # rescaling the reference elevation from [-180 180] to [-def_elevation def_elevation] for scoring purpose test_doa_gt[:, nb_classes:] = test_doa_gt[:, nb_classes:] / ( 180. / def_elevation) test_sed_loss = evaluation_metrics.compute_sed_scores( test_sed_pred, test_sed_gt, data_gen_test.nb_frames_1s()) test_doa_loss = evaluation_metrics.compute_doa_scores_regr( test_doa_pred, test_doa_gt, test_sed_pred, test_sed_gt) test_metric_loss = evaluation_metrics.compute_seld_metric( test_sed_loss, test_doa_loss) avg_scores_test.append([ test_sed_loss[0], test_sed_loss[1], test_doa_loss[0], test_doa_loss[1], test_metric_loss ]) print('Results on test split:') print('\tSELD_score: {}, '.format(test_metric_loss)) print('\tDOA Metrics: DOA_error: {}, frame_recall: {}'.format( test_doa_loss[0], test_doa_loss[1])) print('\tSED Metrics: ER_overall: {}, F1_overall: {}\n'.format( test_sed_loss[0], test_sed_loss[1])) print('\n\nValidation split scores per fold:\n') for cnt in range(len(val_splits)): print( '\tSplit {} - SED ER: {} F1: {}; DOA error: {} frame recall: {}; SELD score: {}' .format(cnt, avg_scores_val[cnt][0], avg_scores_val[cnt][1], avg_scores_val[cnt][2], avg_scores_val[cnt][3], avg_scores_val[cnt][4])) if params['mode'] is 'dev': print('\n\nTesting split scores per fold:\n') for cnt in range(len(val_splits)): print( '\tSplit {} - SED ER: {} F1: {}; DOA error: {} frame recall: {}; SELD score: {}' .format(cnt, avg_scores_test[cnt][0], avg_scores_test[cnt][1], avg_scores_test[cnt][2], avg_scores_test[cnt][3], avg_scores_test[cnt][4]))
def main(argv): """ Main wrapper for training sound event localization and detection network. :param argv: expects two optional inputs. first input: task_id - (optional) To chose the system configuration in parameters.py. (default) 1 - uses default parameters second input: job_id - (optional) all the output files will be uniquely represented with this. (default) 1 """ print(argv) if len(argv) != 3: print('\n\n') print( '-------------------------------------------------------------------------------------------------------' ) print('The code expected two optional inputs') print('\t>> python seld.py <task-id> <job-id>') print( '\t\t<task-id> is used to choose the user-defined parameter set from parameter.py' ) print('Using default inputs for now') print( '\t\t<job-id> is a unique identifier which is used for output filenames (models, training plots). ' 'You can use any number or string for this.') print( '-------------------------------------------------------------------------------------------------------' ) print('\n\n') # use parameter set defined by user task_id = '1' if len(argv) < 2 else argv[1] params = parameter.get_params(task_id) job_id = 1 if len(argv) < 3 else argv[-1] feat_cls = cls_feature_class.FeatureClass(params) train_splits, val_splits, test_splits = None, None, None if params['mode'] == 'dev': test_splits = [1] val_splits = [2] train_splits = [[3, 4, 5, 6]] elif params['mode'] == 'eval': test_splits = [[7, 8]] val_splits = [[1]] train_splits = [[2, 3, 4, 5, 6]] avg_scores_val = [] avg_scores_test = [] for split_cnt, split in enumerate(test_splits): print( '\n\n---------------------------------------------------------------------------------------------------' ) print( '------------------------------------ SPLIT {} -----------------------------------------------' .format(split)) print( '---------------------------------------------------------------------------------------------------' ) # Unique name for the run cls_feature_class.create_folder(params['model_dir']) unique_name = '{}_{}_{}_{}_split{}'.format(task_id, job_id, params['dataset'], params['mode'], split) unique_name = os.path.join(params['model_dir'], unique_name) model_name = '{}_model.h5'.format(unique_name) print("unique_name: {}\n".format(unique_name)) # Load train and validation data print('Loading training dataset:') data_gen_train = cls_data_generator.DataGenerator( params=params, split=train_splits[split_cnt]) print('Loading validation dataset:') data_gen_val = cls_data_generator.DataGenerator( params=params, split=val_splits[split_cnt], shuffle=False) # Collect the reference labels for validation data data_in, data_out = data_gen_train.get_data_sizes() print('FEATURES:\n\tdata_in: {}\n\tdata_out: {}\n'.format( data_in, data_out)) nb_classes = data_gen_train.get_nb_classes() gt = collect_test_labels(data_gen_val, data_out, nb_classes, params['quick_test']) sed_gt = evaluation_metrics.reshape_3Dto2D(gt[0]) doa_gt = evaluation_metrics.reshape_3Dto2D(gt[1]) print( 'MODEL:\n\tdropout_rate: {}\n\tCNN: nb_cnn_filt: {}, f_pool_size{}, t_pool_size{}\n\trnn_size: {}, fnn_size: {}\n\tdoa_objective: {}\n' .format(params['dropout_rate'], params['nb_cnn2d_filt'], params['f_pool_size'], params['t_pool_size'], params['rnn_size'], params['fnn_size'], params['doa_objective'])) print('Using loss weights : {}'.format(params['loss_weights'])) model = keras_model.get_model(data_in=data_in, data_out=data_out, dropout_rate=params['dropout_rate'], nb_cnn2d_filt=params['nb_cnn2d_filt'], f_pool_size=params['f_pool_size'], t_pool_size=params['t_pool_size'], rnn_size=params['rnn_size'], fnn_size=params['fnn_size'], weights=params['loss_weights'], doa_objective=params['doa_objective']) best_seld_metric = 99999 best_epoch = -1 patience_cnt = 0 nb_epoch = 2 if params['quick_test'] else params['nb_epochs'] seld_metric = np.zeros(nb_epoch) new_seld_metric = np.zeros(nb_epoch) tr_loss = np.zeros(nb_epoch) doa_metric = np.zeros((nb_epoch, 6)) sed_metric = np.zeros((nb_epoch, 2)) new_metric = np.zeros((nb_epoch, 4)) # start training for epoch_cnt in range(nb_epoch): start = time.time() # train once per epoch hist = model.fit_generator( generator=data_gen_train.generate(), steps_per_epoch=2 if params['quick_test'] else data_gen_train.get_total_batches_in_data(), epochs=params['epochs_per_fit'], verbose=2, ) tr_loss[epoch_cnt] = hist.history.get('loss')[-1] # predict once per peoch pred = model.predict_generator( generator=data_gen_val.generate(), steps=2 if params['quick_test'] else data_gen_val.get_total_batches_in_data(), verbose=2) sed_pred = evaluation_metrics.reshape_3Dto2D(pred[0]) > 0.5 doa_pred = evaluation_metrics.reshape_3Dto2D( pred[1] if params['doa_objective'] is 'mse' else pred[1][:, :, nb_classes:]) # Calculate the DCASE 2019 metrics - Detection-only and Localization-only scores sed_metric[epoch_cnt, :] = evaluation_metrics.compute_sed_scores( sed_pred, sed_gt, data_gen_val.nb_frames_1s()) doa_metric[ epoch_cnt, :] = evaluation_metrics.compute_doa_scores_regr_xyz( doa_pred, doa_gt, sed_pred, sed_gt) seld_metric[epoch_cnt] = evaluation_metrics.early_stopping_metric( sed_metric[epoch_cnt, :], doa_metric[epoch_cnt, :]) # Calculate the DCASE 2020 metrics - Location-aware detection and Class-aware localization scores cls_new_metric = SELD_evaluation_metrics.SELDMetrics( nb_classes=data_gen_val.get_nb_classes(), doa_threshold=params['lad_doa_thresh']) pred_dict = feat_cls.regression_label_format_to_output_format( sed_pred, doa_pred) gt_dict = feat_cls.regression_label_format_to_output_format( sed_gt, doa_gt) pred_blocks_dict = feat_cls.segment_labels(pred_dict, sed_pred.shape[0]) gt_blocks_dict = feat_cls.segment_labels(gt_dict, sed_gt.shape[0]) cls_new_metric.update_seld_scores_xyz(pred_blocks_dict, gt_blocks_dict) new_metric[epoch_cnt, :] = cls_new_metric.compute_seld_scores() new_seld_metric[ epoch_cnt] = evaluation_metrics.early_stopping_metric( new_metric[epoch_cnt, :2], new_metric[epoch_cnt, 2:]) # Visualize the metrics with respect to epochs plot_functions(unique_name, tr_loss, sed_metric, doa_metric, seld_metric, new_metric, new_seld_metric) patience_cnt += 1 if new_seld_metric[epoch_cnt] < best_seld_metric: best_seld_metric = new_seld_metric[epoch_cnt] best_epoch = epoch_cnt model.save(model_name) patience_cnt = 0 print( 'epoch_cnt: {}, time: {:0.2f}s, tr_loss: {:0.2f}, ' '\n\t\t DCASE2019 SCORES: ER: {:0.2f}, F: {:0.1f}, DE: {:0.1f}, FR:{:0.1f}, seld_score: {:0.2f}, ' '\n\t\t DCASE2020 SCORES: ER: {:0.2f}, F: {:0.1f}, DE: {:0.1f}, DE_F:{:0.1f}, seld_score (early stopping score): {:0.2f}, ' 'best_seld_score: {:0.2f}, best_epoch : {}\n'.format( epoch_cnt, time.time() - start, tr_loss[epoch_cnt], sed_metric[epoch_cnt, 0], sed_metric[epoch_cnt, 1] * 100, doa_metric[epoch_cnt, 0], doa_metric[epoch_cnt, 1] * 100, seld_metric[epoch_cnt], new_metric[epoch_cnt, 0], new_metric[epoch_cnt, 1] * 100, new_metric[epoch_cnt, 2], new_metric[epoch_cnt, 3] * 100, new_seld_metric[epoch_cnt], best_seld_metric, best_epoch)) if patience_cnt > params['patience']: break avg_scores_val.append([ new_metric[best_epoch, 0], new_metric[best_epoch, 1], new_metric[best_epoch, 2], new_metric[best_epoch, 3], best_seld_metric ]) print('\nResults on validation split:') print('\tUnique_name: {} '.format(unique_name)) print('\tSaved model for the best_epoch: {}'.format(best_epoch)) print('\tSELD_score (early stopping score) : {}'.format( best_seld_metric)) print('\n\tDCASE2020 scores') print( '\tClass-aware localization scores: DOA_error: {:0.1f}, F-score: {:0.1f}' .format(new_metric[best_epoch, 2], new_metric[best_epoch, 3] * 100)) print( '\tLocation-aware detection scores: Error rate: {:0.2f}, F-score: {:0.1f}' .format(new_metric[best_epoch, 0], new_metric[best_epoch, 1] * 100)) print('\n\tDCASE2019 scores') print( '\tLocalization-only scores: DOA_error: {:0.1f}, Frame recall: {:0.1f}' .format(doa_metric[best_epoch, 0], doa_metric[best_epoch, 1] * 100)) print( '\tDetection-only scores: Error rate: {:0.2f}, F-score: {:0.1f}\n'. format(sed_metric[best_epoch, 0], sed_metric[best_epoch, 1] * 100)) # ------------------ Calculate metric scores for unseen test split --------------------------------- print( '\nLoading the best model and predicting results on the testing split' ) print('\tLoading testing dataset:') data_gen_test = cls_data_generator.DataGenerator( params=params, split=split, shuffle=False, per_file=params['dcase_output'], is_eval=True if params['mode'] is 'eval' else False) model = keras_model.load_seld_model('{}_model.h5'.format(unique_name), params['doa_objective']) pred_test = model.predict_generator( generator=data_gen_test.generate(), steps=2 if params['quick_test'] else data_gen_test.get_total_batches_in_data(), verbose=2) test_sed_pred = evaluation_metrics.reshape_3Dto2D(pred_test[0]) > 0.5 test_doa_pred = evaluation_metrics.reshape_3Dto2D( pred_test[1] if params['doa_objective'] is 'mse' else pred_test[1][:, :, nb_classes:]) if params['dcase_output']: # Dump results in DCASE output format for calculating final scores dcase_dump_folder = os.path.join( params['dcase_dir'], '{}_{}_{}'.format(task_id, params['dataset'], params['mode'])) cls_feature_class.create_folder(dcase_dump_folder) print('Dumping recording-wise results in: {}'.format( dcase_dump_folder)) test_filelist = data_gen_test.get_filelist() # Number of frames for a 60 second audio with 20ms hop length = 3000 frames max_frames_with_content = data_gen_test.get_nb_frames() # Number of frames in one batch (batch_size* sequence_length) consists of all the 3000 frames above with # zero padding in the remaining frames frames_per_file = data_gen_test.get_frame_per_file() for file_cnt in range(test_sed_pred.shape[0] // frames_per_file): output_file = os.path.join( dcase_dump_folder, test_filelist[file_cnt].replace('.npy', '.csv')) dc = file_cnt * frames_per_file output_dict = feat_cls.regression_label_format_to_output_format( test_sed_pred[dc:dc + max_frames_with_content, :], test_doa_pred[dc:dc + max_frames_with_content, :]) data_gen_test.write_output_format_file(output_file, output_dict) if params['mode'] is 'dev': test_data_in, test_data_out = data_gen_test.get_data_sizes() test_gt = collect_test_labels(data_gen_test, test_data_out, nb_classes, params['quick_test']) test_sed_gt = evaluation_metrics.reshape_3Dto2D(test_gt[0]) test_doa_gt = evaluation_metrics.reshape_3Dto2D(test_gt[1]) # Calculate DCASE2019 scores test_sed_loss = evaluation_metrics.compute_sed_scores( test_sed_pred, test_sed_gt, data_gen_test.nb_frames_1s()) test_doa_loss = evaluation_metrics.compute_doa_scores_regr_xyz( test_doa_pred, test_doa_gt, test_sed_pred, test_sed_gt) test_metric_loss = evaluation_metrics.early_stopping_metric( test_sed_loss, test_doa_loss) # Calculate DCASE2020 scores cls_new_metric = SELD_evaluation_metrics.SELDMetrics( nb_classes=data_gen_test.get_nb_classes(), doa_threshold=20) test_pred_dict = feat_cls.regression_label_format_to_output_format( test_sed_pred, test_doa_pred) test_gt_dict = feat_cls.regression_label_format_to_output_format( test_sed_gt, test_doa_gt) test_pred_blocks_dict = feat_cls.segment_labels( test_pred_dict, test_sed_pred.shape[0]) test_gt_blocks_dict = feat_cls.segment_labels( test_gt_dict, test_sed_gt.shape[0]) cls_new_metric.update_seld_scores_xyz(test_pred_blocks_dict, test_gt_blocks_dict) test_new_metric = cls_new_metric.compute_seld_scores() test_new_seld_metric = evaluation_metrics.early_stopping_metric( test_new_metric[:2], test_new_metric[2:]) avg_scores_test.append([ test_new_metric[0], test_new_metric[1], test_new_metric[2], test_new_metric[3], test_new_seld_metric ]) print('Results on test split:') print('\tDCASE2020 Scores') print( '\tClass-aware localization scores: DOA Error: {:0.1f}, F-score: {:0.1f}' .format(test_new_metric[2], test_new_metric[3] * 100)) print( '\tLocation-aware detection scores: Error rate: {:0.2f}, F-score: {:0.1f}' .format(test_new_metric[0], test_new_metric[1] * 100)) print('\tSELD (early stopping metric): {:0.2f}'.format( test_new_seld_metric)) print('\n\tDCASE2019 Scores') print( '\tLocalization-only scores: DOA Error: {:0.1f}, Frame recall: {:0.1f}' .format(test_doa_loss[0], test_doa_loss[1] * 100)) print( '\tDetection-only scores:Error rate: {:0.2f}, F-score: {:0.1f}' .format(test_sed_loss[0], test_sed_loss[1] * 100))
def main(argv): """ Main wrapper for training sound event localization and detection network. :param argv: expects two optional inputs. first input: job_id - (optional) all the output files will be uniquely represented with this. (default) 1 second input: task_id - (optional) To chose the system configuration in parameters.py. (default) uses default parameters """ if len(argv) != 3: print('\n\n') print('-------------------------------------------------------------------------------------------------------') print('The code expected two inputs') print('\t>> python seld.py <job-id> <task-id>') print('\t\t<job-id> is a unique identifier which is used for output filenames (models, training plots). ' 'You can use any number or string for this.') print('\t\t<task-id> is used to choose the user-defined parameter set from parameter.py') print('Using default inputs for now') print('-------------------------------------------------------------------------------------------------------') print('\n\n') # use parameter set defined by user task_id = '1' if len(argv) < 3 else argv[-1] params = parameter.get_params(task_id) job_id = 1 if len(argv) < 2 else argv[1] model_dir = 'models/' utils.create_folder(model_dir) unique_name = '{}_train{}_validation{}_seq{}'.format(params['dataset'], params['train_split'], params['val_split'], params['sequence_length']) unique_name = os.path.join(model_dir, unique_name) print("unique_name: {}\n".format(unique_name)) data_gen_train = cls_data_generator.DataGenerator( dataset=params['dataset'], ov=params['overlap'], split=params['train_split'], db=params['db'], nfft=params['nfft'], batch_size=params['batch_size'], seq_len=params['sequence_length'], classifier_mode=params['mode'], weakness=params['weakness'], datagen_mode='train', cnn3d=params['cnn_3d'], xyz_def_zero=params['xyz_def_zero'], azi_only=params['azi_only'] ) data_gen_test = cls_data_generator.DataGenerator( dataset=params['dataset'], ov=params['overlap'], split=params['val_split'], db=params['db'], nfft=params['nfft'], batch_size=params['batch_size'], seq_len=params['sequence_length'], classifier_mode=params['mode'], weakness=params['weakness'], datagen_mode='validation', cnn3d=params['cnn_3d'], xyz_def_zero=params['xyz_def_zero'], azi_only=params['azi_only'], shuffle=False ) data_in, data_out = data_gen_train.get_data_sizes() #n_classes = data_out[0][2] print( 'FEATURES:\n' '\tdata_in: {}\n' '\tdata_out: {}\n'.format( data_in, data_out ) ) gt = collect_test_labels(data_gen_test, data_out, params['mode'], params['quick_test']) sed_gt = evaluation_metrics.reshape_3Dto2D(gt[0]) doa_gt = evaluation_metrics.reshape_3Dto2D(gt[1]) print( 'MODEL:\n' '\tdropout_rate: {}\n' '\tCNN: nb_cnn_filt: {}, pool_size{}\n' '\trnn_size: {}, fnn_size: {}\n'.format( params['dropout_rate'], params['nb_cnn3d_filt'] if params['cnn_3d'] else params['nb_cnn2d_filt'], params['pool_size'], params['rnn_size'], params['fnn_size'] ) ) model = keras_model.get_model(data_in=data_in, data_out=data_out, dropout_rate=params['dropout_rate'], nb_cnn2d_filt=params['nb_cnn2d_filt'], pool_size=params['pool_size'], rnn_size=params['rnn_size'], fnn_size=params['fnn_size'], classification_mode=params['mode'], weights=params['loss_weights'], summary=True) if (os.path.exists('{}_model.ckpt'.format(unique_name))): print("Model found!") model.load_weights('{}_model.ckpt'.format(unique_name)) for i in range(10): print("###") best_metric = 99999 conf_mat = None best_conf_mat = None best_epoch = -1 patience_cnt = 0 epoch_metric_loss = np.zeros(params['nb_epochs']) sed_score = np.zeros(params['nb_epochs']) doa_score = np.zeros(params['nb_epochs']) seld_score = np.zeros(params['nb_epochs']) tr_loss = np.zeros(params['nb_epochs']) val_loss = np.zeros(params['nb_epochs']) doa_loss = np.zeros((params['nb_epochs'], 6)) sed_loss = np.zeros((params['nb_epochs'], 2)) for epoch_cnt in range(params['nb_epochs']): start = time.time() print("##### Training the model #####") hist = model.fit_generator( generator=data_gen_train.generate(), steps_per_epoch=params['quick_test_steps'] if params[ 'quick_test'] else data_gen_train.get_total_batches_in_data(), validation_data=data_gen_test.generate(), validation_steps=params['quick_test_steps'] if params[ 'quick_test'] else data_gen_test.get_total_batches_in_data(), use_multiprocessing=False, workers=1, epochs=1, verbose=1 ) tr_loss[epoch_cnt] = hist.history.get('loss')[-1] val_loss[epoch_cnt] = hist.history.get('val_loss')[-1] print("##########################") # Save, get model and re-load weights for the predict_generator bug print("##### Saving weights #####") model.save_weights('{}_model.ckpt'.format(unique_name)) model = keras_model.get_model(data_in=data_in, data_out=data_out, dropout_rate=params['dropout_rate'], nb_cnn2d_filt=params['nb_cnn2d_filt'], pool_size=params['pool_size'], rnn_size=params['rnn_size'], fnn_size=params['fnn_size'], classification_mode=params['mode'], weights=params['loss_weights'], summary=False) model.load_weights('{}_model.ckpt'.format(unique_name)) print("##########################") print("#### Prediction on validation split ####") pred = model.predict_generator( generator=data_gen_test.generate(), steps=params['quick_test_steps'] if params['quick_test'] else data_gen_test.get_total_batches_in_data(), use_multiprocessing=False, workers=1, verbose=1 ) print("########################################") # print("pred:",pred[1].shape) if params['mode'] == 'regr': sed_pred = np.array(evaluation_metrics.reshape_3Dto2D(pred[0])) > .5 doa_pred = evaluation_metrics.reshape_3Dto2D(pred[1]) # Old confidence intervals ''' sed_err = sed_gt - sed_pred [sed_conf_low, sed_conf_up, sed_median] = compute_confidence(sed_err) # print("Condidence Interval for SED error is [" + str(sed_conf_low) + ", " + str(sed_conf_up) + "]") print("Confidence Interval for SED error is [ %.5f, %.5f ]" % (sed_conf_low, sed_conf_up)) # print("\tMedian is " + str(sed_median)) print("\tMedian is %.5f" % (sed_median)) # print("\tDisplacement: +/- " + str(sed_conf_up - sed_median)) print("\tDisplacement: +/- %.5f" % (sed_conf_up - sed_median)) doa_err = doa_gt - doa_pred [doa_conf_low, doa_conf_up, doa_median] = compute_confidence(doa_err) # print("Condidence Interval for DOA is [" + str(doa_conf_low) + ", " + str(doa_conf_up) + "]") print("Confidence Interval for DOA is [ %.5f, %.5f ]" % (doa_conf_low, doa_conf_up)) # print("Median is " + str(doa_median)) print("\tMedian is %.5f" % (doa_median)) # print("Displacement: +/- " + str(doa_conf_up - doa_median)) print("\tDisplacement: +/- %.5f" % (doa_conf_up - doa_median)) ''' sed_loss[epoch_cnt, :] = evaluation_metrics.compute_sed_scores(sed_pred, sed_gt, data_gen_test.nb_frames_1s()) if params['azi_only']: doa_loss[epoch_cnt, :], conf_mat = evaluation_metrics.compute_doa_scores_regr_xy(doa_pred, doa_gt, sed_pred, sed_gt) else: doa_loss[epoch_cnt, :], conf_mat = evaluation_metrics.compute_doa_scores_regr_xyz(doa_pred, doa_gt, sed_pred, sed_gt) sed_score[epoch_cnt] = np.mean([sed_loss[epoch_cnt, 0], 1 - sed_loss[epoch_cnt, 1]]) doa_score[epoch_cnt] = np.mean([2 * np.arcsin(doa_loss[epoch_cnt, 1] / 2.0) / np.pi, 1 - (doa_loss[epoch_cnt, 5] / float(doa_gt.shape[0]))]) seld_score[epoch_cnt] = (sed_score[epoch_cnt] + doa_score[epoch_cnt]) / 2 if os.path.isdir('./models'): plot.imshow(conf_mat, cmap='binary', interpolation='None') plot.savefig('models/confusion_matrix.jpg') # New confidence computation, differing doa and sed errors sed_err = sed_loss[epoch_cnt, 0] [sed_conf_low, sed_conf_up] = compute_confidence(sed_err, sed_pred.shape[0]) print("Confidence Interval for SED error is [ %f, %f ]" % (sed_conf_low, sed_conf_up)) #doa_err = doa_gt - doa_pred #[x_err, y_err, z_err] = compute_doa_confidence(doa_err, n_classes) plot_array = [tr_loss[epoch_cnt], # 0 val_loss[epoch_cnt], # 1 sed_loss[epoch_cnt][0], # 2 er sed_loss[epoch_cnt][1], # 3 f1 doa_loss[epoch_cnt][0], # 4 avg_accuracy doa_loss[epoch_cnt][1], # 5 doa_loss_gt doa_loss[epoch_cnt][2], # 6 doa_loss_pred doa_loss[epoch_cnt][3], # 7 doa_loss_gt_cnt doa_loss[epoch_cnt][4], # 8 doa_loss_pred_cnt doa_loss[epoch_cnt][5], # 9 good_frame_cnt sed_score[epoch_cnt], # 10 doa_score[epoch_cnt], seld_score[epoch_cnt], #doa_conf_low, doa_median, #doa_conf_up, sed_conf_low, #sed_median, sed_conf_up] sed_conf_low, sed_conf_up] patience_cnt += 1 # model.save_weights('{}_model.ckpt'.format(unique_name)) simple_plotter.save_array_to_csv("{}_plot.csv".format(unique_name), plot_array) #simple_plotter.plot_confidence(x_err, y_err, z_err, "ov") print("##### Model and metrics saved! #####") if seld_score[epoch_cnt] < best_metric: best_metric = seld_score[epoch_cnt] best_conf_mat = conf_mat best_epoch = epoch_cnt # Now we save the model at every iteration model.save_weights('{}_BEST_model.ckpt'.format(unique_name)) patience_cnt = 0 print('epoch_cnt: %d, time: %.2fs, tr_loss: %.4f, val_loss: %.4f, ' 'F1_overall: %.2f, ER_overall: %.2f, ' 'doa_error_gt: %.2f, doa_error_pred: %.2f, good_pks_ratio:%.2f, ' 'sed_score: %.4f, doa_score: %.4f, seld_score: %.4f, best_error_metric: %.2f, best_epoch : %d' % ( epoch_cnt, time.time() - start, tr_loss[epoch_cnt], val_loss[epoch_cnt], sed_loss[epoch_cnt, 1], sed_loss[epoch_cnt, 0], doa_loss[epoch_cnt, 1], doa_loss[epoch_cnt, 2], doa_loss[epoch_cnt, 5] / float(sed_gt.shape[0]), sed_score[epoch_cnt], doa_score[epoch_cnt], seld_score[epoch_cnt], best_metric, best_epoch ) ) # plot_functions(unique_name, tr_loss, val_loss, sed_loss, doa_loss, sed_score, doa_score, epoch_cnt) print('best_conf_mat : {}'.format(best_conf_mat)) print('best_conf_mat_diag : {}'.format(np.diag(best_conf_mat))) print('saved model for the best_epoch: {} with best_metric: {}, '.format(best_epoch, best_metric)) print('DOA Metrics: doa_loss_gt: {}, doa_loss_pred: {}, good_pks_ratio: {}'.format( doa_loss[best_epoch, 1], doa_loss[best_epoch, 2], doa_loss[best_epoch, 5] / float(sed_gt.shape[0]))) print('SED Metrics: ER_overall: {}, F1_overall: {}'.format(sed_loss[best_epoch, 0], sed_loss[best_epoch, 1])) print('unique_name: {} '.format(unique_name))