def test_model(state, reference_tsv_path, reduced_number_of_data=None, strore_predicitions_fname=None): dataset = DatasetDcase2019Task4(os.path.join(cfg.workspace), base_feature_dir=os.path.join( cfg.workspace, "dataset", "features"), save_log_feature=False) crnn_kwargs = state["model"]["kwargs"] crnn = CRNN(**crnn_kwargs) crnn.load(parameters=state["model"]["state_dict"]) LOG.info("Model loaded at epoch: {}".format(state["epoch"])) pooling_time_ratio = state["pooling_time_ratio"] crnn.load(parameters=state["model"]["state_dict"]) scaler = Scaler() scaler.load_state_dict(state["scaler"]) classes = cfg.classes many_hot_encoder = ManyHotEncoder.load_state_dict( state["many_hot_encoder"]) crnn = crnn.eval() [crnn] = to_cuda_if_available([crnn]) transforms_valid = get_transforms(cfg.max_frames, scaler=scaler) LOG.info(reference_tsv_path) df = dataset.initialize_and_get_df(reference_tsv_path, reduced_number_of_data) strong_dataload = DataLoadDf(df, dataset.get_feature_file, many_hot_encoder.encode_strong_df, transform=transforms_valid) predictions = get_predictions(crnn, strong_dataload, many_hot_encoder.decode_strong, pooling_time_ratio, save_predictions=strore_predicitions_fname) compute_strong_metrics(predictions, df) weak_dataload = DataLoadDf(df, dataset.get_feature_file, many_hot_encoder.encode_weak, transform=transforms_valid) weak_metric = get_f_measure_by_class( crnn, len(classes), DataLoader(weak_dataload, batch_size=cfg.batch_size)) LOG.info("Weak F1-score per class: \n {}".format( pd.DataFrame(weak_metric * 100, many_hot_encoder.labels))) LOG.info("Weak F1-score macro averaged: {}".format(np.mean(weak_metric)))
def train_classifier(train_loader, classif_model, optimizer_classif, many_hot_encoder=None, valid_loader=None, state={}, dir_model="model", result_path="res", recompute=True): criterion_bce = nn.BCELoss() classif_model, criterion_bce = to_cuda_if_available(classif_model, criterion_bce) print(classif_model) early_stopping_call = EarlyStopping(patience=cfg.early_stopping, val_comp="sup", init_patience=cfg.first_early_wait) save_best_call = SaveBest(val_comp="sup") # scheduler = ReduceLROnPlateau(optimizer_classif, 'max', factor=0.1, patience=cfg.reduce_lr, # verbose=True) print(optimizer_classif) save_results = pd.DataFrame() create_folder(dir_model) if cfg.save_best: model_path_sup1 = os.path.join(dir_model, "best_model") else: model_path_sup1 = os.path.join(dir_model, "epoch_" + str(cfg.n_epoch_classifier)) print("path of model : " + model_path_sup1) state['many_hot_encoder'] = many_hot_encoder.state_dict() if not os.path.exists(model_path_sup1) or recompute: for epoch_ in range(cfg.n_epoch_classifier): print(classif_model.training) start = time.time() loss_mean_bce = [] for i, samples in enumerate(train_loader): inputs, pred_labels = samples if i == 0: LOG.debug("classif input shape: {}".format(inputs.shape)) # zero the parameter gradients optimizer_classif.zero_grad() inputs = to_cuda_if_available(inputs) # forward + backward + optimize weak_out = classif_model(inputs) weak_out = to_cpu(weak_out) # print(output) loss_bce = criterion_bce(weak_out, pred_labels) loss_mean_bce.append(loss_bce.item()) loss_bce.backward() optimizer_classif.step() loss_mean_bce = np.mean(loss_mean_bce) classif_model.eval() n_class = len(many_hot_encoder.labels) macro_f_measure_train = get_f_measure_by_class(classif_model, n_class, train_loader) if valid_loader is not None: macro_f_measure = get_f_measure_by_class(classif_model, n_class, valid_loader) mean_macro_f_measure = np.mean(macro_f_measure) else: mean_macro_f_measure = -1 classif_model.train() print("Time to train an epoch: {}".format(time.time() - start)) # print statistics print('[%d / %d, %5d] loss: %.3f' % (epoch_ + 1, cfg.n_epoch_classifier, i + 1, loss_mean_bce)) results = {"train_loss": loss_mean_bce, "macro_measure_train": np.mean(macro_f_measure_train), "class_macro_train": np.array_str(macro_f_measure_train, precision=2), "macro_measure_valid": mean_macro_f_measure, "class_macro_valid": np.array_str(macro_f_measure, precision=2), } for key in results: LOG.info("\t\t ----> {} : {}".format(key, results[key])) save_results = save_results.append(results, ignore_index=True) # scheduler.step(mean_macro_f_measure) # ########## # # Callbacks # ########## state['epoch'] = epoch_ + 1 state["model"]["state_dict"] = classif_model.state_dict() state["optimizer"]["state_dict"] = optimizer_classif.state_dict() state["loss"] = loss_mean_bce state.update(results) if cfg.early_stopping is not None: if early_stopping_call.apply(mean_macro_f_measure): print("EARLY STOPPING") break if cfg.save_best and save_best_call.apply(mean_macro_f_measure): save_model(state, model_path_sup1) if cfg.save_best: LOG.info( "best model at epoch : {} with macro {}".format(save_best_call.best_epoch, save_best_call.best_val)) LOG.info("loading model from: {}".format(model_path_sup1)) classif_model, state = load_model(model_path_sup1, return_optimizer=False, return_state=True) else: model_path_sup1 = os.path.join(dir_model, "epoch_" + str(cfg.n_epoch_classifier)) save_model(state, model_path_sup1) LOG.debug("model path: {}".format(model_path_sup1)) LOG.debug('Finished Training') else: classif_model, state = load_model(model_path_sup1, return_optimizer=False, return_state=True) LOG.info("#### End classif") save_results.to_csv(result_path, sep="\t", header=True, index=False) return classif_model, state
def test_model(state, reduced_number_of_data, strore_predicitions_fname=None): crnn_kwargs = state["model"]["kwargs"] crnn = CRNN(**crnn_kwargs) crnn.load(parameters=state["model"]["state_dict"]) LOG.info("Model loaded at epoch: {}".format(state["epoch"])) pooling_time_ratio = state["pooling_time_ratio"] crnn.load(parameters=state["model"]["state_dict"]) scaler = Scaler() scaler.load_state_dict(state["scaler"]) classes = cfg.classes many_hot_encoder = ManyHotEncoder.load_state_dict( state["many_hot_encoder"]) # ############## # Validation # ############## crnn = crnn.eval() [crnn] = to_cuda_if_available([crnn]) transforms_valid = get_transforms(cfg.max_frames, scaler=scaler) # # 2018 # LOG.info("Eval 2018") # eval_2018_df = dataset.initialize_and_get_df(cfg.eval2018, reduced_number_of_data) # # Strong # eval_2018_strong = DataLoadDf(eval_2018_df, dataset.get_feature_file, many_hot_encoder.encode_strong_df, # transform=transforms_valid) # predictions = get_predictions(crnn, eval_2018_strong, many_hot_encoder.decode_strong) # compute_strong_metrics(predictions, eval_2018_df, pooling_time_ratio) # # Weak # eval_2018_weak = DataLoadDf(eval_2018_df, dataset.get_feature_file, many_hot_encoder.encode_weak, # transform=transforms_valid) # weak_metric = get_f_measure_by_class(crnn, len(classes), DataLoader(eval_2018_weak, batch_size=cfg.batch_size)) # LOG.info("Weak F1-score per class: \n {}".format(pd.DataFrame(weak_metric * 100, many_hot_encoder.labels))) # LOG.info("Weak F1-score macro averaged: {}".format(np.mean(weak_metric))) # Validation 2019 # LOG.info("Validation 2019 (original code)") # b_dataset = B_DatasetDcase2019Task4(cfg.workspace, # base_feature_dir=os.path.join(cfg.workspace, 'dataset', 'features'), # save_log_feature=False) # b_validation_df = b_dataset.initialize_and_get_df(cfg.validation, reduced_number_of_data) # b_validation_df.to_csv('old.csv') # b_validation_strong = B_DataLoadDf(b_validation_df, # b_dataset.get_feature_file, many_hot_encoder.encode_strong_df, # transform=transforms_valid) # predictions2 = get_predictions(crnn, b_validation_strong, many_hot_encoder.decode_strong, # save_predictions=strore_predicitions_fname) # compute_strong_metrics(predictions2, b_validation_df, pooling_time_ratio) # b_validation_weak = B_DataLoadDf(b_validation_df, b_dataset.get_feature_file, many_hot_encoder.encode_weak, # transform=transforms_valid) # weak_metric = get_f_measure_by_class(crnn, len(classes), DataLoader(b_validation_weak, batch_size=cfg.batch_size)) # LOG.info("Weak F1-score per class: \n {}".format(pd.DataFrame(weak_metric * 100, many_hot_encoder.labels))) # LOG.info("Weak F1-score macro averaged: {}".format(np.mean(weak_metric))) # ============================================================================================ # ============================================================================================ # ============================================================================================ dataset = DatasetDcase2019Task4(feature_dir=cfg.feature_dir, local_path=cfg.workspace, exp_tag=cfg.exp_tag, save_log_feature=False) # Validation 2019 LOG.info("Validation 2019") validation_df = dataset.initialize_and_get_df(cfg.validation, reduced_number_of_data) validation_strong = DataLoadDf(validation_df, dataset.get_feature_file, many_hot_encoder.encode_strong_df, transform=transforms_valid) predictions = get_predictions(crnn, validation_strong, many_hot_encoder.decode_strong, save_predictions=strore_predicitions_fname) vdf = validation_df.copy() vdf.filename = vdf.filename.str.replace('.npy', '.wav') pdf = predictions.copy() pdf.filename = pdf.filename.str.replace('.npy', '.wav') compute_strong_metrics(pdf, vdf, pooling_time_ratio) validation_weak = DataLoadDf(validation_df, dataset.get_feature_file, many_hot_encoder.encode_weak, transform=transforms_valid) weak_metric = get_f_measure_by_class( crnn, len(classes), DataLoader(validation_weak, batch_size=cfg.batch_size)) LOG.info("Weak F1-score per class: \n {}".format( pd.DataFrame(weak_metric * 100, many_hot_encoder.labels))) LOG.info("Weak F1-score macro averaged: {}".format(np.mean(weak_metric)))
train(training_data, crnn, optimizer, epoch, weak_mask, strong_mask) crnn = crnn.eval() LOG.info("Training synthetic metric:") train_predictions = get_predictions(crnn, train_synth_data, many_hot_encoder.decode_strong, pooling_time_ratio, save_predictions=None) train_metric = compute_strong_metrics(train_predictions, train_synth_df) if not no_weak: LOG.info("Training weak metric:") weak_metric = get_f_measure_by_class( crnn, len(classes), DataLoader(train_weak_data, batch_size=cfg.batch_size)) LOG.info("Weak F1-score per class: \n {}".format( pd.DataFrame(weak_metric * 100, many_hot_encoder.labels))) LOG.info("Weak F1-score macro averaged: {}".format( np.mean(weak_metric))) LOG.info("Valid weak metric:") weak_metric = get_f_measure_by_class( crnn, len(classes), DataLoader(valid_weak_data, batch_size=cfg.batch_size)) LOG.info("Weak F1-score per class: \n {}".format( pd.DataFrame(weak_metric * 100, many_hot_encoder.labels))) LOG.info("Weak F1-score macro averaged: {}".format( np.mean(weak_metric)))
def main(parameters): log = dcase_util.ui.ui.FancyLogger() log.title('DCASE2018 / Task4') overwirte_preprocessing = False overwrite_learning = True overwrite_testing = True overwrite_evaluation = True # ===================================================================== # Parameters # ===================================================================== # Process parameters param = dcase_util.containers.DCASEAppParameterContainer( parameters, path_structure={ 'FEATURE_EXTRACTOR': ['DATASET', 'FEATURE_EXTRACTOR'], 'FEATURE_NORMALIZER': ['DATASET', 'FEATURE_EXTRACTOR'], 'LEARNER': [ 'DATASET', 'FEATURE_EXTRACTOR', 'FEATURE_NORMALIZER', 'FEATURE_SEQUENCER', 'LEARNER' ], 'RECOGNIZER': [ 'DATASET', 'FEATURE_EXTRACTOR', 'FEATURE_NORMALIZER', 'FEATURE_SEQUENCER', 'LEARNER', 'RECOGNIZER' ], }).process() # Make sure all system paths exists dcase_util.utils.Path().create(paths=list(param['path'].values())) # Initialize keras_model_first_pass = None keras_model_second_pass = None # ===================================================================== # Dataset # ===================================================================== # Get dataset and initialize it db = DCASE2018_Task4_DevelopmentSet( included_content_types=['all'], local_path="", data_path=param.get_path('path.dataset'), audio_paths=[ os.path.join("dataset", "audio", "train", "weak"), os.path.join("dataset", "audio", "train", "unlabel_in_domain"), os.path.join("dataset", "audio", "train", "unlabel_out_of_domain"), os.path.join("dataset", "audio", "test"), os.path.join("dataset", "audio", "eval") ]).initialize() # Active folds folds = db.folds(mode=param.get_path('dataset.parameters.evaluation_mode')) active_fold_list = param.get_path('dataset.parameters.fold_list') if active_fold_list: folds = list(set(folds).intersection(active_fold_list)) # ===================================================================== # Feature extraction stage # ===================================================================== if param.get_path('flow.feature_extraction'): log.section_header('Feature Extraction / Train material') # Prepare feature extractor mel_extractor = dcase_util.features.MelExtractor( **param.get_path('feature_extractor.parameters.mel')) # Loop over all audio files in the dataset and extract features for them. # for audio_filename in db.audio_files: for audio_filename in db.audio_files: # Get filename for feature data from audio filename feature_filename = dcase_util.utils.Path( path=audio_filename).modify(path_base=param.get_path( 'path.application.feature_extractor'), filename_extension='.cpickle') if not os.path.isfile(feature_filename) or overwirte_preprocessing: log.line(data=os.path.split(audio_filename)[1], indent=2) # Load audio data audio = dcase_util.containers.AudioContainer().load( filename=audio_filename, mono=True, fs=param.get_path('feature_extractor.fs')) # Extract features and store them into FeatureContainer, and save it to the disk dcase_util.containers.FeatureContainer( data=mel_extractor.extract(audio.data), time_resolution=param.get_path( 'feature_extractor.hop_length_seconds')).save( filename=feature_filename) log.foot() # ===================================================================== # Feature normalization stage # ===================================================================== if param.get_path('flow.feature_normalization'): log.section_header('Feature Normalization') # Get filename for the normalization factors features_norm_filename = os.path.join( param.get_path('path.application.feature_normalizer'), 'normalize_values.cpickle') if not os.path.isfile( features_norm_filename) or overwirte_preprocessing: normalizer = dcase_util.data.Normalizer( filename=features_norm_filename) # Loop through all training data, two train folds for fold in folds: for filename in db.train(fold=fold).unique_files: # Get feature filename feature_filename = dcase_util.utils.Path( path=filename).modify( path_base=param.get_path( 'path.application.feature_extractor'), filename_extension='.cpickle', ) # Load feature matrix features = dcase_util.containers.FeatureContainer().load( filename=feature_filename) # Accumulate statistics normalizer.accumulate(data=features.data) # Finalize and save normalizer.finalize().save() log.foot() # Create processing chain for features feature_processing_chain = dcase_util.processors.ProcessingChain() for chain in param.get_path('feature_processing_chain'): processor_name = chain.get('processor_name') init_parameters = chain.get('init_parameters', {}) # Inject parameters if processor_name == 'dcase_util.processors.NormalizationProcessor': init_parameters['filename'] = features_norm_filename if init_parameters.get('enable') is None or init_parameters.get( 'enable') is True: feature_processing_chain.push_processor( processor_name=processor_name, init_parameters=init_parameters, ) # ===================================================================== # Learning stage # ===================================================================== if param.get_path('flow.learning'): log.section_header('Learning') # setup keras parameters dcase_util.keras.setup_keras( seed=param.get_path('learner.parameters.random_seed'), profile=param.get_path('learner.parameters.keras_profile'), backend=param.get_path('learner.parameters.backend'), device=param.get_path('learner.parameters.device'), verbose=False) # encoder used to convert text labels into vector many_hot_encoder = dcase_util.data.ManyHotEncoder(label_list=db.tags(), time_resolution=1) # ===================================================================== # Training first pass # ===================================================================== fold = 1 # Get model filename fold1_model_filename = os.path.join( param.get_path('path.application.learner'), 'model_fold_{fold}.h5'.format(fold=fold)) if not os.path.isfile(fold1_model_filename) or overwrite_learning: # Split the dataset into training and validation files training_files, validation_files = db.validation_split( fold=fold, split_type='random', validation_amount=param.get_path( 'learner.parameters.model.first_pass.validation_amount'), verbose=True) batch_size = param.get_path( 'learner.parameters.model.first_pass.fit.batch_size') shuffle = param.get_path( 'learner.parameters.model.first_pass.fit.shuffle') # Get items (with labels) associated with training files training_items = db.train(fold=fold).filter( file_list=training_files) # Create the generator, which convert filename and item into arrays batch_X, batch_y in right formats training_generator = data_generator( training_items, param.get_path('path.application.feature_extractor'), many_hot_encoder, feature_processing_chain, batch_size=batch_size, shuffle=shuffle) validation_items = db.train(fold=fold).filter( file_list=validation_files) validation_generator = data_generator( validation_items, param.get_path('path.application.feature_extractor'), many_hot_encoder, feature_processing_chain, batch_size=batch_size, shuffle=False) # Update constants with useful information to setup the model model_parameter_constants = { 'NB_CLASSES': db.tag_count(), 'INPUT_FREQUENCIES': param.get_path('feature_extractor.parameters.mel.n_mels'), 'INPUT_SEQUENCE_LENGTH': param.get_path('feature_sequencer.sequence_length'), } model_parameter_constants.update( param.get_path('learner.parameters.model.constants', {})) # Load the sequential keras model defined in the YAML. keras_model_first_pass = dcase_util.keras.create_sequential_model( model_parameter_list=param.get_path( 'learner.parameters.model.first_pass.config'), constants=model_parameter_constants) # Print the model configuration keras_model_first_pass.summary(print_fn=log.line) # Create optimizer object from info given in YAML param.set_path(path='learner.parameters.compile.optimizer', new_value=dcase_util.keras.create_optimizer( class_name=param.get_path( 'learner.parameters.optimizer.class_name'), config=param.get_path( 'learner.parameters.optimizer.config'))) # Compile model keras_model_first_pass.compile( **param.get_path('learner.parameters.compile')) epochs = param.get_path( 'learner.parameters.model.first_pass.fit.epochs') # Setup callbacks used during training callback_list = [ dcase_util.keras.ProgressLoggerCallback( epochs=epochs, metric=param.get_path( 'learner.parameters.compile.metrics')[0], loss=param.get_path('learner.parameters.compile.loss'), output_type='logging', **param.get_path( 'learner.parameters.callbacks.ProgressLoggerCallback')) ] if param.get_path('learner.parameters.callbacks.StopperCallback'): callback_list.append( dcase_util.keras.StopperCallback( epochs=epochs, **param.get_path( 'learner.parameters.callbacks.StopperCallback'))) if param.get_path('learner.parameters.callbacks.StasherCallback'): callback_list.append( dcase_util.keras.StasherCallback( epochs=epochs, **param.get_path( 'learner.parameters.callbacks.StasherCallback'))) processing_interval = param.get_path( 'learner.parameters.callbacks.ProgressLoggerCallback.processing_interval' ) epochs = param.get_path( 'learner.parameters.model.first_pass.fit.epochs') # Iterate through epoch to be able to manually update callbacks for epoch_start in range(0, epochs, processing_interval): epoch_end = epoch_start + processing_interval # Make sure we have only specified amount of epochs if epoch_end > epochs: epoch_end = epochs # Train keras_model_first_pass keras_model_first_pass.fit_generator( generator=training_generator, steps_per_epoch=len(training_files) // batch_size, validation_data=validation_generator, validation_steps=len(validation_files) // batch_size, callbacks=callback_list, verbose=0, initial_epoch=epoch_start, epochs=epoch_end) # Get f_measures of the current epoch val_macro_f_measure = get_f_measure_by_class( keras_model_first_pass, db.tag_count(), validation_generator, len(validation_files) // batch_size) val_macro_f_measure = val_macro_f_measure.mean() tra_macro_f_measure = get_f_measure_by_class( keras_model_first_pass, db.tag_count(), training_generator, len(training_files) // batch_size, ) tra_macro_f_measure = tra_macro_f_measure.mean() # Inject external metric values to the callbacks for callback in callback_list: if hasattr(callback, 'set_external_metric_value'): callback.set_external_metric_value( metric_label='val_macro_f_measure', metric_value=val_macro_f_measure) callback.set_external_metric_value( metric_label='tra_macro_f_measure', metric_value=tra_macro_f_measure) # Manually update callbacks for callback in callback_list: if hasattr(callback, 'update'): callback.update() # Check we need to stop training stop_training = False for callback in callback_list: if hasattr(callback, 'stop'): if callback.stop(): log.line("Early stropping") stop_training = True if stop_training: # Stop the training loop break # Fetch best model for callback in callback_list: if isinstance(callback, dcase_util.keras.StasherCallback): callback.log() best_weights = callback.get_best()['weights'] if best_weights: keras_model_first_pass.set_weights(best_weights) break # Save trained model keras_model_first_pass.save(fold1_model_filename) log.foot() # ======= # Calculate best thresholds # ======= thresholds_filename = os.path.join( param.get_path('path.application.learner'), 'thresholds_{fold}.p'.format(fold=fold)) if not os.path.isfile(thresholds_filename) or overwrite_learning: training_files, validation_files = db.validation_split( fold=fold, split_type='random', validation_amount=param.get_path( 'learner.parameters.model.first_pass.validation_amount'), verbose=True) batch_size = param.get_path( 'learner.parameters.model.first_pass.fit.batch_size') validation_items = db.train(fold=fold).filter( file_list=validation_files) validation_generator = data_generator( validation_items, param.get_path('path.application.feature_extractor'), many_hot_encoder, feature_processing_chain, batch_size=batch_size, shuffle=False) # Load model if not trained during this run if not keras_model_first_pass: keras_model_first_pass = keras.models.load_model( fold1_model_filename) thresholds = [0] * db.tag_count() max_f_measure = [-numpy.inf] * db.tag_count() for threshold in numpy.arange(0., 1 + 1e-6, 0.1): # Assign current threshold to each class current_thresholds = [threshold] * db.tag_count() # Calculate f_measures with the current thresholds macro_f_measure = get_f_measure_by_class( keras_model_first_pass, db.tag_count(), validation_generator, len(validation_files) // batch_size, current_thresholds) # Update thresholds for class with better f_measures for i, label in enumerate(db.tags()): f_measure = macro_f_measure[i] if f_measure > max_f_measure[i]: max_f_measure[i] = f_measure thresholds[i] = threshold for i, label in enumerate(db.tags()): log.line("{:30}, threshold: {}".format(label, thresholds[i])) thresholds_filename = os.path.join( param.get_path('path.application.learner'), 'thresholds.p'.format(fold=fold)) pickle.dump(thresholds, open(thresholds_filename, "wb")) else: thresholds = pickle.load(open(thresholds_filename, "rb")) # ===================================================================== # Predict stage from weak to predict unlabel_in_domain tags # ===================================================================== log.section_header( 'Predict 1st pass, add labels to unlabel_in_domain data') # Get results filename eval_predictions_filename = os.path.join( param.get_path('path.application.recognizer'), 'pred_weak_fold_{fold}.txt'.format(fold=fold)) if not os.path.isfile(eval_predictions_filename) or overwrite_testing: # Initialize results container eval_cont = dcase_util.containers.MetaDataContainer( filename=eval_predictions_filename) # Load model if not yet loaded if not keras_model_first_pass: keras_model_first_pass = keras.models.load_model( fold1_model_filename) # Loop through all test files from the current cross-validation fold for item in db.test(fold=fold): # Get feature filename feature_filename = dcase_util.utils.Path( path=item.filename).modify(path_base=param.get_path( 'path.application.feature_extractor'), filename_extension='.cpickle') features = feature_processing_chain.process( filename=feature_filename) input_data = features.data.reshape( features.shape[:-1]).T # (500, 64) input_data = input_data.reshape( (1, ) + input_data.shape) # (1, 500, 64) # Get network output probabilities = keras_model_first_pass.predict(x=input_data) # Binarization of the network output frame_decisions = dcase_util.data.ProbabilityEncoder( ).binarization(probabilities=probabilities, binarization_type='class_threshold', threshold=thresholds, time_axis=0) estimated_tags = dcase_util.data.DecisionEncoder( label_list=db.tags()).many_hot( frame_decisions=frame_decisions, time_axis=0) # Store result into results container eval_cont.append({ 'filename': item.filename, 'tags': estimated_tags[0] }) # Save results container eval_cont.save() log.foot() # ===================================================================== # Learning stage 2nd pass, learn from weak and unlabel_in_domain annotated data # ===================================================================== fold = 2 log.line(data='Fold [{fold}]'.format(fold=fold), indent=2) # Get model filename fold2_model_filename = os.path.join( param.get_path('path.application.learner'), 'model_fold_{fold}.h5'.format(fold=fold)) if not os.path.isfile(fold2_model_filename) or overwrite_learning: model_parameter_constants = { 'NB_CLASSES': db.tag_count(), 'INPUT_FREQUENCIES': param.get_path('feature_extractor.parameters.mel.n_mels'), 'INPUT_SEQUENCE_LENGTH': param.get_path('feature_sequencer.sequence_length'), } model_parameter_constants.update( param.get_path('learner.parameters.model.constants', {})) keras_model_second_pass = dcase_util.keras.create_sequential_model( model_parameter_list=param.get_path( 'learner.parameters.model.second_pass.config'), constants=model_parameter_constants) keras_model_second_pass.summary(print_fn=log.line) # Create optimizer object param.set_path(path='learner.parameters.compile.optimizer', new_value=dcase_util.keras.create_optimizer( class_name=param.get_path( 'learner.parameters.optimizer.class_name'), config=param.get_path( 'learner.parameters.optimizer.config'))) # Compile model keras_model_second_pass.compile( **param.get_path('learner.parameters.compile')) # Get annotations from the 1st pass model fold1_results_filename = os.path.join( param.get_path('path.application.recognizer'), 'pred_weak_fold_{fold}.txt'.format(fold=1)) # Load annotations predictions_first_pass = dcase_util.containers.MetaDataContainer( filename=fold1_results_filename).load() # Split the dataset into train and validation. If "weak" is provided, files from weak.csv are used to # validate the model. Else, give a percentage which will be used if param.get_path( 'learner.parameters.model.second_pass.validation_amount' ) == "weak": training_files = predictions_first_pass.unique_files training_items = predictions_first_pass validation_files = db.train(fold=1).unique_files validation_items = db.train(fold=1) else: # Get validation files training_files, validation_files = db.validation_split( fold=fold, split_type='random', validation_amount=param.get_path( 'learner.parameters.model.second_pass.validation_amount' ), verbose=False) training_fold2 = predictions_first_pass + db.train(fold=1) training_items = training_fold2.filter( file_list=training_files) validation_items = training_fold2.filter( file_list=validation_files) processing_interval = param.get_path( 'learner.parameters.callbacks.ProgressLoggerCallback.processing_interval' ) epochs = param.get_path( 'learner.parameters.model.second_pass.fit.epochs') batch_size = param.get_path( 'learner.parameters.model.second_pass.fit.batch_size') shuffle = param.get_path( 'learner.parameters.model.second_pass.fit.shuffle') # Create generators, which convert filename and item into arrays batch_X, batch_y in right formats training_generator = data_generator( training_items, param.get_path('path.application.feature_extractor'), many_hot_encoder, feature_processing_chain, batch_size=batch_size, shuffle=shuffle, mode="strong") validation_generator = data_generator( validation_items, param.get_path('path.application.feature_extractor'), many_hot_encoder, feature_processing_chain, batch_size=batch_size, shuffle=False, mode="strong") # Initialize callbacks used during training callback_list = [ dcase_util.keras.ProgressLoggerCallback( epochs=param.get_path( 'learner.parameters.model.second_pass.fit.epochs'), metric=param.get_path( 'learner.parameters.compile.metrics')[0], loss=param.get_path('learner.parameters.compile.loss'), output_type='logging', **param.get_path( 'learner.parameters.callbacks.ProgressLoggerCallback')) ] if param.get_path('learner.parameters.callbacks.StopperCallback'): callback_list.append( dcase_util.keras.StopperCallback( epochs=param.get_path( 'learner.parameters.model.second_pass.fit.epochs'), **param.get_path( 'learner.parameters.callbacks.StopperCallback'))) if param.get_path('learner.parameters.callbacks.StasherCallback'): callback_list.append( dcase_util.keras.StasherCallback( epochs=param.get_path( 'learner.parameters.model.second_pass.fit.epochs'), **param.get_path( 'learner.parameters.callbacks.StasherCallback'))) for epoch_start in range(0, epochs, processing_interval): epoch_end = epoch_start + processing_interval # Make sure we have only specified amount of epochs if epoch_end > epochs: epoch_end = epochs # Train keras_model_second_pass keras_model_second_pass.fit_generator( generator=training_generator, steps_per_epoch=len(training_files) // batch_size, validation_data=validation_generator, validation_steps=len(validation_files) // batch_size, callbacks=callback_list, verbose=0, initial_epoch=epoch_start, epochs=epoch_end) # Calculate external metrics, f_measure of the current epoch val_macro_f_measure = get_f_measure_by_class( keras_model_second_pass, db.tag_count(), validation_generator, len(validation_files) // batch_size, ) val_macro_f_measure = val_macro_f_measure.mean() tra_macro_f_measure = get_f_measure_by_class( keras_model_second_pass, db.tag_count(), training_generator, len(training_files) // batch_size, ) tra_macro_f_measure = tra_macro_f_measure.mean() # Inject external metric values to the callbacks for callback in callback_list: if hasattr(callback, 'set_external_metric_value'): callback.set_external_metric_value( metric_label='val_macro_f_measure', metric_value=val_macro_f_measure) callback.set_external_metric_value( metric_label='tra_macro_f_measure', metric_value=tra_macro_f_measure) # Manually update callbacks for callback in callback_list: if hasattr(callback, 'update'): callback.update() # Check we need to stop training stop_training = False for callback in callback_list: if hasattr(callback, 'stop'): if callback.stop(): log.line("Early stropping") stop_training = True if stop_training: # Stop the training loop break # Fetch best model for callback in callback_list: if isinstance(callback, dcase_util.keras.StasherCallback): callback.log() best_weights = callback.get_best()['weights'] if best_weights: keras_model_second_pass.set_weights(best_weights) break # Save trained model keras_model_second_pass.save(fold2_model_filename) log.foot() # ===================================================================== # Testing stage, get strong annotations # ===================================================================== fold = 2 if param.get_path('flow.testing'): log.section_header('Testing') # Get results filename test_predictions_filename = os.path.join( param.get_path('path.application.recognizer'), 'test_fold_{fold}.csv'.format(fold=2)) # Get model filename fold2_model_filename = os.path.join( param.get_path('path.application.learner'), 'res_fold_{fold}.h5'.format(fold=fold)) if not os.path.isfile(test_predictions_filename) or overwrite_testing: # Load model if not yet loaded if not keras_model_second_pass: keras_model_second_pass = keras.models.load_model( "/home/nturpault/model_fold_2.h5") # Initialize results container test_meta = dcase_util.containers.MetaDataContainer( filename=test_predictions_filename) # Loop through all test files from the current cross-validation fold for item in db.test(fold=fold): # Get feature filename feature_filename = dcase_util.utils.Path( path=item.filename).modify(path_base=param.get_path( 'path.application.feature_extractor'), filename_extension='.cpickle') # Get features array features = feature_processing_chain.process( filename=feature_filename) input_data = features.data.reshape( features.shape[:-1]).T # (500, 64) # Create a batch with only one file input_data = input_data.reshape( (1, ) + input_data.shape) # (1, 500, 64) # Get network output for strong data probabilities = keras_model_second_pass.predict(input_data) # only one file in the batch probabilities = probabilities[0] if param.get_path('recognizer.frame_binarization.enable'): # Binarization of the network output frame_decisions = dcase_util.data.ProbabilityEncoder( ).binarization( probabilities=probabilities, binarization_type=param.get_path( 'recognizer.frame_binarization.binarization_type'), threshold=param.get_path( 'recognizer.frame_binarization.threshold'), time_axis=0) else: frame_decisions = dcase_util.data.ProbabilityEncoder( ).binarization(probabilities=probabilities, binarization_type="global_threshold", threshold=0.5, time_axis=0) decision_encoder = dcase_util.data.DecisionEncoder( label_list=db.tags()) if param.get_path('recognizer.process_activity.enable'): frame_decisions = decision_encoder.process_activity( frame_decisions, window_length=param.get_path( 'recognizer.process_activity.window_length'), time_axis=0) for i, label in enumerate(db.tags()): # given a list of ones, give the onset and offset in frames estimated_events = decision_encoder.find_contiguous_regions( activity_array=frame_decisions[:, i]) for [onset, offset] in estimated_events: hop_length_seconds = param.get_path( 'feature_extractor.hop_length_seconds') # Store result into results container, convert frames to seconds test_meta.append({ 'filename': item.filename, 'event_label': label, 'onset': onset * hop_length_seconds, 'offset': offset * hop_length_seconds }) # Save results container test_meta.save() stats_filename = os.path.join( param.get_path('path.application.recognizer'), 'test_results.txt') if not os.path.isfile(stats_filename) or overwrite_testing: # test data used to evaluate the system reference_event_list = db.test(fold=fold) # Evaluation done on dataframes reference_df = pandas.DataFrame(reference_event_list) # predictions done during the step test before eval_df = pandas.read_csv(test_predictions_filename, sep="\t", header=0) # Calculate the metric event_based_metric = event_based_evaluation_df( reference_df, eval_df) with open(stats_filename, "w") as stats_file: stats_file.write(event_based_metric.__str__()) log.line(event_based_metric.__str__(), indent=4) log.foot() # ===================================================================== # Evaluation stage, predict evaluation labels # ===================================================================== if param.get_path('flow.evaluation'): log.section_header('Evaluation') # Get results filename eval_predictions_filename = os.path.join( param.get_path('path.application.recognizer'), 'eval_results.csv') # Get model filename fold2_model_filename = os.path.join( param.get_path('path.application.learner'), 'model_fold_{fold}.h5'.format(fold=2)) if not os.path.isfile( eval_predictions_filename) or overwrite_evaluation: # Load model if not yet loaded if not keras_model_second_pass: keras_model_second_pass = keras.models.load_model( fold2_model_filename) # Initialize results container eval_meta = dcase_util.containers.MetaDataContainer( filename=eval_predictions_filename) # Loop through all test files from the current cross-validation fold for item in db.eval(fold=2): # Get feature filename feature_filename = dcase_util.utils.Path( path=item.filename).modify(path_base=param.get_path( 'path.application.feature_extractor'), filename_extension='.cpickle') # Get features array features = feature_processing_chain.process( filename=feature_filename) input_data = features.data.reshape( features.shape[:-1]).T # (500, 64) # Create a batch with only one file input_data = input_data.reshape( (1, ) + input_data.shape) # (1, 500, 64) # Get network output for strong data probabilities = keras_model_second_pass.predict(input_data) # only one file in the batch probabilities = probabilities[0] if param.get_path('recognizer.frame_binarization.enable'): # Binarization of the network output frame_decisions = dcase_util.data.ProbabilityEncoder( ).binarization( probabilities=probabilities, binarization_type=param.get_path( 'recognizer.frame_binarization.binarization_type'), threshold=param.get_path( 'recognizer.frame_binarization.threshold'), time_axis=0) else: frame_decisions = dcase_util.data.ProbabilityEncoder( ).binarization(probabilities=probabilities, binarization_type="global_threshold", threshold=0.5, time_axis=0) decision_encoder = dcase_util.data.DecisionEncoder( label_list=db.tags()) if param.get_path('recognizer.process_activity.enable'): frame_decisions = decision_encoder.process_activity( frame_decisions, window_length=param.get_path( 'recognizer.process_activity.window_length'), time_axis=0) for i, label in enumerate(db.tags()): # given a list of ones, give the onset and offset in frames estimated_events = decision_encoder.find_contiguous_regions( activity_array=frame_decisions[:, i]) for [onset, offset] in estimated_events: hop_length_seconds = param.get_path( 'feature_extractor.hop_length_seconds') # Store result into results container, convert frames to seconds eval_meta.append({ 'filename': item.filename, 'event_label': label, 'onset': onset * hop_length_seconds, 'offset': offset * hop_length_seconds }) # Save results container eval_meta.save() log.line("Find the eval predictions in : " + eval_predictions_filename) stats_filename = os.path.join( param.get_path('path.application.recognizer'), 'eval_results.txt') if not os.path.isfile(stats_filename) or overwrite_evaluation: # get the two dataframes with events and evaluate the system reference_event_list = db.eval(fold=2) reference_df = pandas.DataFrame(reference_event_list) eval_df = pandas.read_csv(eval_predictions_filename, sep="\t", header=0) # Calculate the metric event_based_metric = event_based_evaluation_df( reference_df, eval_df) with open(stats_filename, "w") as stats_file: stats_file.write(event_based_metric.__str__()) log.line(event_based_metric.__str__(), indent=4) log.foot()
loss_bce = criterion_bce(weak_out, pred_labels) loss_bce.backward() loss_bce.append(loss_bce.item()) optimizer.step() loss_bce = np.mean(loss_bce) print('[%d / %d, %5d] loss: %.3f' % (epoch_ + 1, n_epochs, cnt + 1, loss_bce)) return loss_bce, model if not os.path.exists(model_path_sup1) or cfg.recompute_classif: for epoch_ in range(n_epochs): start = time.time() loss_mean_bce, model = train_loop(train_load, model) model.eval() macro_f_measure_train = get_f_measure_by_class( model, len(classes), train_set_emb, max=args.single_label) macro_f_measure_val = get_f_measure_by_class(model, len(classes), valid_set_val, max=args.single_label) mean_macro_f_measure = np.mean(macro_f_measure_val) macro_f_measure_test = get_f_measure_by_class( model, len(classes), test_set_val, max=args.single_label) model.train() print("Time to train an epoch: {}".format(time.time() - start)) # print statistics results = { "train_loss":
epoch, ema_model=crnn_ema, weak_mask=weak_mask, strong_mask=strong_mask) crnn = crnn.eval() LOG.info("\n ### Validation Metrics ### \n") # predictions = get_predictions(crnn, validation_data, many_hot_encoder.decode_strong, # save_predictions=None) # pdf = predictions.copy() # pdf.filename = pdf.filename.str.replace('.npy', '.wav') # valid_events_metric = compute_strong_metrics(pdf, valid_synth_df, pooling_time_ratio) # LOG.info("\n ### Valid weak metric ### \n") weak_metric = get_f_measure_by_class( crnn, len(cfg.classes), DataLoader(validation_data, batch_size=cfg.batch_size)) LOG.info("Weak F1-score per class: \n {}".format( pd.DataFrame(weak_metric * 100, many_hot_encoder.labels))) LOG.info("Weak F1-score macro averaged: {}".format( np.mean(weak_metric))) state['model']['state_dict'] = crnn.state_dict() state['model_ema']['state_dict'] = crnn_ema.state_dict() state['optimizer']['state_dict'] = optimizer.state_dict() state['epoch'] = epoch # state['valid_metric'] = valid_events_metric.results() if cfg.checkpoint_epochs is not None and ( epoch + 1) % cfg.checkpoint_epochs == 0: model_fname = os.path.join(saved_model_dir, '_epoch_' + str(epoch))