def test_model(state,
reference_tsv_path,
reduced_number_of_data=None,
strore_predicitions_fname=None):
dataset = DatasetDcase2019Task4(os.path.join(cfg.workspace),
base_feature_dir=os.path.join(
cfg.workspace, "dataset", "features"),
save_log_feature=False)
crnn_kwargs = state["model"]["kwargs"]
crnn = CRNN(**crnn_kwargs)
crnn.load(parameters=state["model"]["state_dict"])
LOG.info("Model loaded at epoch: {}".format(state["epoch"]))
pooling_time_ratio = state["pooling_time_ratio"]
crnn.load(parameters=state["model"]["state_dict"])
scaler = Scaler()
scaler.load_state_dict(state["scaler"])
classes = cfg.classes
many_hot_encoder = ManyHotEncoder.load_state_dict(
state["many_hot_encoder"])
crnn = crnn.eval()
[crnn] = to_cuda_if_available([crnn])
transforms_valid = get_transforms(cfg.max_frames, scaler=scaler)
LOG.info(reference_tsv_path)
df = dataset.initialize_and_get_df(reference_tsv_path,
reduced_number_of_data)
strong_dataload = DataLoadDf(df,
dataset.get_feature_file,
many_hot_encoder.encode_strong_df,
transform=transforms_valid)
predictions = get_predictions(crnn,
strong_dataload,
many_hot_encoder.decode_strong,
pooling_time_ratio,
save_predictions=strore_predicitions_fname)
compute_strong_metrics(predictions, df)
weak_dataload = DataLoadDf(df,
dataset.get_feature_file,
many_hot_encoder.encode_weak,
transform=transforms_valid)
weak_metric = get_f_measure_by_class(
crnn, len(classes), DataLoader(weak_dataload,
batch_size=cfg.batch_size))
LOG.info("Weak F1-score per class: \n {}".format(
pd.DataFrame(weak_metric * 100, many_hot_encoder.labels)))
LOG.info("Weak F1-score macro averaged: {}".format(np.mean(weak_metric)))
def train_classifier(train_loader, classif_model, optimizer_classif, many_hot_encoder=None,
valid_loader=None, state={},
dir_model="model", result_path="res", recompute=True):
criterion_bce = nn.BCELoss()
classif_model, criterion_bce = to_cuda_if_available(classif_model, criterion_bce)
print(classif_model)
early_stopping_call = EarlyStopping(patience=cfg.early_stopping, val_comp="sup",
init_patience=cfg.first_early_wait)
save_best_call = SaveBest(val_comp="sup")
# scheduler = ReduceLROnPlateau(optimizer_classif, 'max', factor=0.1, patience=cfg.reduce_lr,
# verbose=True)
print(optimizer_classif)
save_results = pd.DataFrame()
create_folder(dir_model)
if cfg.save_best:
model_path_sup1 = os.path.join(dir_model, "best_model")
else:
model_path_sup1 = os.path.join(dir_model, "epoch_" + str(cfg.n_epoch_classifier))
print("path of model : " + model_path_sup1)
state['many_hot_encoder'] = many_hot_encoder.state_dict()
if not os.path.exists(model_path_sup1) or recompute:
for epoch_ in range(cfg.n_epoch_classifier):
print(classif_model.training)
start = time.time()
loss_mean_bce = []
for i, samples in enumerate(train_loader):
inputs, pred_labels = samples
if i == 0:
LOG.debug("classif input shape: {}".format(inputs.shape))
# zero the parameter gradients
optimizer_classif.zero_grad()
inputs = to_cuda_if_available(inputs)
# forward + backward + optimize
weak_out = classif_model(inputs)
weak_out = to_cpu(weak_out)
# print(output)
loss_bce = criterion_bce(weak_out, pred_labels)
loss_mean_bce.append(loss_bce.item())
loss_bce.backward()
optimizer_classif.step()
loss_mean_bce = np.mean(loss_mean_bce)
classif_model.eval()
n_class = len(many_hot_encoder.labels)
macro_f_measure_train = get_f_measure_by_class(classif_model, n_class,
train_loader)
if valid_loader is not None:
macro_f_measure = get_f_measure_by_class(classif_model, n_class,
valid_loader)
mean_macro_f_measure = np.mean(macro_f_measure)
else:
mean_macro_f_measure = -1
classif_model.train()
print("Time to train an epoch: {}".format(time.time() - start))
# print statistics
print('[%d / %d, %5d] loss: %.3f' %
(epoch_ + 1, cfg.n_epoch_classifier, i + 1, loss_mean_bce))
results = {"train_loss": loss_mean_bce,
"macro_measure_train": np.mean(macro_f_measure_train),
"class_macro_train": np.array_str(macro_f_measure_train, precision=2),
"macro_measure_valid": mean_macro_f_measure,
"class_macro_valid": np.array_str(macro_f_measure, precision=2),
}
for key in results:
LOG.info("\t\t ----> {} : {}".format(key, results[key]))
save_results = save_results.append(results, ignore_index=True)
# scheduler.step(mean_macro_f_measure)
# ##########
# # Callbacks
# ##########
state['epoch'] = epoch_ + 1
state["model"]["state_dict"] = classif_model.state_dict()
state["optimizer"]["state_dict"] = optimizer_classif.state_dict()
state["loss"] = loss_mean_bce
state.update(results)
if cfg.early_stopping is not None:
if early_stopping_call.apply(mean_macro_f_measure):
print("EARLY STOPPING")
break
if cfg.save_best and save_best_call.apply(mean_macro_f_measure):
save_model(state, model_path_sup1)
if cfg.save_best:
LOG.info(
"best model at epoch : {} with macro {}".format(save_best_call.best_epoch, save_best_call.best_val))
LOG.info("loading model from: {}".format(model_path_sup1))
classif_model, state = load_model(model_path_sup1, return_optimizer=False, return_state=True)
else:
model_path_sup1 = os.path.join(dir_model, "epoch_" + str(cfg.n_epoch_classifier))
save_model(state, model_path_sup1)
LOG.debug("model path: {}".format(model_path_sup1))
LOG.debug('Finished Training')
else:
classif_model, state = load_model(model_path_sup1, return_optimizer=False, return_state=True)
LOG.info("#### End classif")
save_results.to_csv(result_path, sep="\t", header=True, index=False)
return classif_model, state
def test_model(state, reduced_number_of_data, strore_predicitions_fname=None):
crnn_kwargs = state["model"]["kwargs"]
crnn = CRNN(**crnn_kwargs)
crnn.load(parameters=state["model"]["state_dict"])
LOG.info("Model loaded at epoch: {}".format(state["epoch"]))
pooling_time_ratio = state["pooling_time_ratio"]
crnn.load(parameters=state["model"]["state_dict"])
scaler = Scaler()
scaler.load_state_dict(state["scaler"])
classes = cfg.classes
many_hot_encoder = ManyHotEncoder.load_state_dict(
state["many_hot_encoder"])
# ##############
# Validation
# ##############
crnn = crnn.eval()
[crnn] = to_cuda_if_available([crnn])
transforms_valid = get_transforms(cfg.max_frames, scaler=scaler)
# # 2018
# LOG.info("Eval 2018")
# eval_2018_df = dataset.initialize_and_get_df(cfg.eval2018, reduced_number_of_data)
# # Strong
# eval_2018_strong = DataLoadDf(eval_2018_df, dataset.get_feature_file, many_hot_encoder.encode_strong_df,
# transform=transforms_valid)
# predictions = get_predictions(crnn, eval_2018_strong, many_hot_encoder.decode_strong)
# compute_strong_metrics(predictions, eval_2018_df, pooling_time_ratio)
# # Weak
# eval_2018_weak = DataLoadDf(eval_2018_df, dataset.get_feature_file, many_hot_encoder.encode_weak,
# transform=transforms_valid)
# weak_metric = get_f_measure_by_class(crnn, len(classes), DataLoader(eval_2018_weak, batch_size=cfg.batch_size))
# LOG.info("Weak F1-score per class: \n {}".format(pd.DataFrame(weak_metric * 100, many_hot_encoder.labels)))
# LOG.info("Weak F1-score macro averaged: {}".format(np.mean(weak_metric)))
# Validation 2019
# LOG.info("Validation 2019 (original code)")
# b_dataset = B_DatasetDcase2019Task4(cfg.workspace,
# base_feature_dir=os.path.join(cfg.workspace, 'dataset', 'features'),
# save_log_feature=False)
# b_validation_df = b_dataset.initialize_and_get_df(cfg.validation, reduced_number_of_data)
# b_validation_df.to_csv('old.csv')
# b_validation_strong = B_DataLoadDf(b_validation_df,
# b_dataset.get_feature_file, many_hot_encoder.encode_strong_df,
# transform=transforms_valid)
# predictions2 = get_predictions(crnn, b_validation_strong, many_hot_encoder.decode_strong,
# save_predictions=strore_predicitions_fname)
# compute_strong_metrics(predictions2, b_validation_df, pooling_time_ratio)
# b_validation_weak = B_DataLoadDf(b_validation_df, b_dataset.get_feature_file, many_hot_encoder.encode_weak,
# transform=transforms_valid)
# weak_metric = get_f_measure_by_class(crnn, len(classes), DataLoader(b_validation_weak, batch_size=cfg.batch_size))
# LOG.info("Weak F1-score per class: \n {}".format(pd.DataFrame(weak_metric * 100, many_hot_encoder.labels)))
# LOG.info("Weak F1-score macro averaged: {}".format(np.mean(weak_metric)))
# ============================================================================================
# ============================================================================================
# ============================================================================================
dataset = DatasetDcase2019Task4(feature_dir=cfg.feature_dir,
local_path=cfg.workspace,
exp_tag=cfg.exp_tag,
save_log_feature=False)
# Validation 2019
LOG.info("Validation 2019")
validation_df = dataset.initialize_and_get_df(cfg.validation,
reduced_number_of_data)
validation_strong = DataLoadDf(validation_df,
dataset.get_feature_file,
many_hot_encoder.encode_strong_df,
transform=transforms_valid)
predictions = get_predictions(crnn,
validation_strong,
many_hot_encoder.decode_strong,
save_predictions=strore_predicitions_fname)
vdf = validation_df.copy()
vdf.filename = vdf.filename.str.replace('.npy', '.wav')
pdf = predictions.copy()
pdf.filename = pdf.filename.str.replace('.npy', '.wav')
compute_strong_metrics(pdf, vdf, pooling_time_ratio)
validation_weak = DataLoadDf(validation_df,
dataset.get_feature_file,
many_hot_encoder.encode_weak,
transform=transforms_valid)
weak_metric = get_f_measure_by_class(
crnn, len(classes),
DataLoader(validation_weak, batch_size=cfg.batch_size))
LOG.info("Weak F1-score per class: \n {}".format(
pd.DataFrame(weak_metric * 100, many_hot_encoder.labels)))
LOG.info("Weak F1-score macro averaged: {}".format(np.mean(weak_metric)))
train(training_data, crnn, optimizer, epoch, weak_mask, strong_mask)
crnn = crnn.eval()
LOG.info("Training synthetic metric:")
train_predictions = get_predictions(crnn,
train_synth_data,
many_hot_encoder.decode_strong,
pooling_time_ratio,
save_predictions=None)
train_metric = compute_strong_metrics(train_predictions,
train_synth_df)
if not no_weak:
LOG.info("Training weak metric:")
weak_metric = get_f_measure_by_class(
crnn, len(classes),
DataLoader(train_weak_data, batch_size=cfg.batch_size))
LOG.info("Weak F1-score per class: \n {}".format(
pd.DataFrame(weak_metric * 100, many_hot_encoder.labels)))
LOG.info("Weak F1-score macro averaged: {}".format(
np.mean(weak_metric)))
LOG.info("Valid weak metric:")
weak_metric = get_f_measure_by_class(
crnn, len(classes),
DataLoader(valid_weak_data, batch_size=cfg.batch_size))
LOG.info("Weak F1-score per class: \n {}".format(
pd.DataFrame(weak_metric * 100, many_hot_encoder.labels)))
LOG.info("Weak F1-score macro averaged: {}".format(
np.mean(weak_metric)))
def main(parameters):
log = dcase_util.ui.ui.FancyLogger()
log.title('DCASE2018 / Task4')
overwirte_preprocessing = False
overwrite_learning = True
overwrite_testing = True
overwrite_evaluation = True
# =====================================================================
# Parameters
# =====================================================================
# Process parameters
param = dcase_util.containers.DCASEAppParameterContainer(
parameters,
path_structure={
'FEATURE_EXTRACTOR': ['DATASET', 'FEATURE_EXTRACTOR'],
'FEATURE_NORMALIZER': ['DATASET', 'FEATURE_EXTRACTOR'],
'LEARNER': [
'DATASET', 'FEATURE_EXTRACTOR', 'FEATURE_NORMALIZER',
'FEATURE_SEQUENCER', 'LEARNER'
],
'RECOGNIZER': [
'DATASET', 'FEATURE_EXTRACTOR', 'FEATURE_NORMALIZER',
'FEATURE_SEQUENCER', 'LEARNER', 'RECOGNIZER'
],
}).process()
# Make sure all system paths exists
dcase_util.utils.Path().create(paths=list(param['path'].values()))
# Initialize
keras_model_first_pass = None
keras_model_second_pass = None
# =====================================================================
# Dataset
# =====================================================================
# Get dataset and initialize it
db = DCASE2018_Task4_DevelopmentSet(
included_content_types=['all'],
local_path="",
data_path=param.get_path('path.dataset'),
audio_paths=[
os.path.join("dataset", "audio", "train", "weak"),
os.path.join("dataset", "audio", "train", "unlabel_in_domain"),
os.path.join("dataset", "audio", "train", "unlabel_out_of_domain"),
os.path.join("dataset", "audio", "test"),
os.path.join("dataset", "audio", "eval")
]).initialize()
# Active folds
folds = db.folds(mode=param.get_path('dataset.parameters.evaluation_mode'))
active_fold_list = param.get_path('dataset.parameters.fold_list')
if active_fold_list:
folds = list(set(folds).intersection(active_fold_list))
# =====================================================================
# Feature extraction stage
# =====================================================================
if param.get_path('flow.feature_extraction'):
log.section_header('Feature Extraction / Train material')
# Prepare feature extractor
mel_extractor = dcase_util.features.MelExtractor(
**param.get_path('feature_extractor.parameters.mel'))
# Loop over all audio files in the dataset and extract features for them.
# for audio_filename in db.audio_files:
for audio_filename in db.audio_files:
# Get filename for feature data from audio filename
feature_filename = dcase_util.utils.Path(
path=audio_filename).modify(path_base=param.get_path(
'path.application.feature_extractor'),
filename_extension='.cpickle')
if not os.path.isfile(feature_filename) or overwirte_preprocessing:
log.line(data=os.path.split(audio_filename)[1], indent=2)
# Load audio data
audio = dcase_util.containers.AudioContainer().load(
filename=audio_filename,
mono=True,
fs=param.get_path('feature_extractor.fs'))
# Extract features and store them into FeatureContainer, and save it to the disk
dcase_util.containers.FeatureContainer(
data=mel_extractor.extract(audio.data),
time_resolution=param.get_path(
'feature_extractor.hop_length_seconds')).save(
filename=feature_filename)
log.foot()
# =====================================================================
# Feature normalization stage
# =====================================================================
if param.get_path('flow.feature_normalization'):
log.section_header('Feature Normalization')
# Get filename for the normalization factors
features_norm_filename = os.path.join(
param.get_path('path.application.feature_normalizer'),
'normalize_values.cpickle')
if not os.path.isfile(
features_norm_filename) or overwirte_preprocessing:
normalizer = dcase_util.data.Normalizer(
filename=features_norm_filename)
# Loop through all training data, two train folds
for fold in folds:
for filename in db.train(fold=fold).unique_files:
# Get feature filename
feature_filename = dcase_util.utils.Path(
path=filename).modify(
path_base=param.get_path(
'path.application.feature_extractor'),
filename_extension='.cpickle',
)
# Load feature matrix
features = dcase_util.containers.FeatureContainer().load(
filename=feature_filename)
# Accumulate statistics
normalizer.accumulate(data=features.data)
# Finalize and save
normalizer.finalize().save()
log.foot()
# Create processing chain for features
feature_processing_chain = dcase_util.processors.ProcessingChain()
for chain in param.get_path('feature_processing_chain'):
processor_name = chain.get('processor_name')
init_parameters = chain.get('init_parameters', {})
# Inject parameters
if processor_name == 'dcase_util.processors.NormalizationProcessor':
init_parameters['filename'] = features_norm_filename
if init_parameters.get('enable') is None or init_parameters.get(
'enable') is True:
feature_processing_chain.push_processor(
processor_name=processor_name,
init_parameters=init_parameters,
)
# =====================================================================
# Learning stage
# =====================================================================
if param.get_path('flow.learning'):
log.section_header('Learning')
# setup keras parameters
dcase_util.keras.setup_keras(
seed=param.get_path('learner.parameters.random_seed'),
profile=param.get_path('learner.parameters.keras_profile'),
backend=param.get_path('learner.parameters.backend'),
device=param.get_path('learner.parameters.device'),
verbose=False)
# encoder used to convert text labels into vector
many_hot_encoder = dcase_util.data.ManyHotEncoder(label_list=db.tags(),
time_resolution=1)
# =====================================================================
# Training first pass
# =====================================================================
fold = 1
# Get model filename
fold1_model_filename = os.path.join(
param.get_path('path.application.learner'),
'model_fold_{fold}.h5'.format(fold=fold))
if not os.path.isfile(fold1_model_filename) or overwrite_learning:
# Split the dataset into training and validation files
training_files, validation_files = db.validation_split(
fold=fold,
split_type='random',
validation_amount=param.get_path(
'learner.parameters.model.first_pass.validation_amount'),
verbose=True)
batch_size = param.get_path(
'learner.parameters.model.first_pass.fit.batch_size')
shuffle = param.get_path(
'learner.parameters.model.first_pass.fit.shuffle')
# Get items (with labels) associated with training files
training_items = db.train(fold=fold).filter(
file_list=training_files)
# Create the generator, which convert filename and item into arrays batch_X, batch_y in right formats
training_generator = data_generator(
training_items,
param.get_path('path.application.feature_extractor'),
many_hot_encoder,
feature_processing_chain,
batch_size=batch_size,
shuffle=shuffle)
validation_items = db.train(fold=fold).filter(
file_list=validation_files)
validation_generator = data_generator(
validation_items,
param.get_path('path.application.feature_extractor'),
many_hot_encoder,
feature_processing_chain,
batch_size=batch_size,
shuffle=False)
# Update constants with useful information to setup the model
model_parameter_constants = {
'NB_CLASSES':
db.tag_count(),
'INPUT_FREQUENCIES':
param.get_path('feature_extractor.parameters.mel.n_mels'),
'INPUT_SEQUENCE_LENGTH':
param.get_path('feature_sequencer.sequence_length'),
}
model_parameter_constants.update(
param.get_path('learner.parameters.model.constants', {}))
# Load the sequential keras model defined in the YAML.
keras_model_first_pass = dcase_util.keras.create_sequential_model(
model_parameter_list=param.get_path(
'learner.parameters.model.first_pass.config'),
constants=model_parameter_constants)
# Print the model configuration
keras_model_first_pass.summary(print_fn=log.line)
# Create optimizer object from info given in YAML
param.set_path(path='learner.parameters.compile.optimizer',
new_value=dcase_util.keras.create_optimizer(
class_name=param.get_path(
'learner.parameters.optimizer.class_name'),
config=param.get_path(
'learner.parameters.optimizer.config')))
# Compile model
keras_model_first_pass.compile(
**param.get_path('learner.parameters.compile'))
epochs = param.get_path(
'learner.parameters.model.first_pass.fit.epochs')
# Setup callbacks used during training
callback_list = [
dcase_util.keras.ProgressLoggerCallback(
epochs=epochs,
metric=param.get_path(
'learner.parameters.compile.metrics')[0],
loss=param.get_path('learner.parameters.compile.loss'),
output_type='logging',
**param.get_path(
'learner.parameters.callbacks.ProgressLoggerCallback'))
]
if param.get_path('learner.parameters.callbacks.StopperCallback'):
callback_list.append(
dcase_util.keras.StopperCallback(
epochs=epochs,
**param.get_path(
'learner.parameters.callbacks.StopperCallback')))
if param.get_path('learner.parameters.callbacks.StasherCallback'):
callback_list.append(
dcase_util.keras.StasherCallback(
epochs=epochs,
**param.get_path(
'learner.parameters.callbacks.StasherCallback')))
processing_interval = param.get_path(
'learner.parameters.callbacks.ProgressLoggerCallback.processing_interval'
)
epochs = param.get_path(
'learner.parameters.model.first_pass.fit.epochs')
# Iterate through epoch to be able to manually update callbacks
for epoch_start in range(0, epochs, processing_interval):
epoch_end = epoch_start + processing_interval
# Make sure we have only specified amount of epochs
if epoch_end > epochs:
epoch_end = epochs
# Train keras_model_first_pass
keras_model_first_pass.fit_generator(
generator=training_generator,
steps_per_epoch=len(training_files) // batch_size,
validation_data=validation_generator,
validation_steps=len(validation_files) // batch_size,
callbacks=callback_list,
verbose=0,
initial_epoch=epoch_start,
epochs=epoch_end)
# Get f_measures of the current epoch
val_macro_f_measure = get_f_measure_by_class(
keras_model_first_pass, db.tag_count(),
validation_generator,
len(validation_files) // batch_size)
val_macro_f_measure = val_macro_f_measure.mean()
tra_macro_f_measure = get_f_measure_by_class(
keras_model_first_pass,
db.tag_count(),
training_generator,
len(training_files) // batch_size,
)
tra_macro_f_measure = tra_macro_f_measure.mean()
# Inject external metric values to the callbacks
for callback in callback_list:
if hasattr(callback, 'set_external_metric_value'):
callback.set_external_metric_value(
metric_label='val_macro_f_measure',
metric_value=val_macro_f_measure)
callback.set_external_metric_value(
metric_label='tra_macro_f_measure',
metric_value=tra_macro_f_measure)
# Manually update callbacks
for callback in callback_list:
if hasattr(callback, 'update'):
callback.update()
# Check we need to stop training
stop_training = False
for callback in callback_list:
if hasattr(callback, 'stop'):
if callback.stop():
log.line("Early stropping")
stop_training = True
if stop_training:
# Stop the training loop
break
# Fetch best model
for callback in callback_list:
if isinstance(callback, dcase_util.keras.StasherCallback):
callback.log()
best_weights = callback.get_best()['weights']
if best_weights:
keras_model_first_pass.set_weights(best_weights)
break
# Save trained model
keras_model_first_pass.save(fold1_model_filename)
log.foot()
# =======
# Calculate best thresholds
# =======
thresholds_filename = os.path.join(
param.get_path('path.application.learner'),
'thresholds_{fold}.p'.format(fold=fold))
if not os.path.isfile(thresholds_filename) or overwrite_learning:
training_files, validation_files = db.validation_split(
fold=fold,
split_type='random',
validation_amount=param.get_path(
'learner.parameters.model.first_pass.validation_amount'),
verbose=True)
batch_size = param.get_path(
'learner.parameters.model.first_pass.fit.batch_size')
validation_items = db.train(fold=fold).filter(
file_list=validation_files)
validation_generator = data_generator(
validation_items,
param.get_path('path.application.feature_extractor'),
many_hot_encoder,
feature_processing_chain,
batch_size=batch_size,
shuffle=False)
# Load model if not trained during this run
if not keras_model_first_pass:
keras_model_first_pass = keras.models.load_model(
fold1_model_filename)
thresholds = [0] * db.tag_count()
max_f_measure = [-numpy.inf] * db.tag_count()
for threshold in numpy.arange(0., 1 + 1e-6, 0.1):
# Assign current threshold to each class
current_thresholds = [threshold] * db.tag_count()
# Calculate f_measures with the current thresholds
macro_f_measure = get_f_measure_by_class(
keras_model_first_pass, db.tag_count(),
validation_generator,
len(validation_files) // batch_size, current_thresholds)
# Update thresholds for class with better f_measures
for i, label in enumerate(db.tags()):
f_measure = macro_f_measure[i]
if f_measure > max_f_measure[i]:
max_f_measure[i] = f_measure
thresholds[i] = threshold
for i, label in enumerate(db.tags()):
log.line("{:30}, threshold: {}".format(label, thresholds[i]))
thresholds_filename = os.path.join(
param.get_path('path.application.learner'),
'thresholds.p'.format(fold=fold))
pickle.dump(thresholds, open(thresholds_filename, "wb"))
else:
thresholds = pickle.load(open(thresholds_filename, "rb"))
# =====================================================================
# Predict stage from weak to predict unlabel_in_domain tags
# =====================================================================
log.section_header(
'Predict 1st pass, add labels to unlabel_in_domain data')
# Get results filename
eval_predictions_filename = os.path.join(
param.get_path('path.application.recognizer'),
'pred_weak_fold_{fold}.txt'.format(fold=fold))
if not os.path.isfile(eval_predictions_filename) or overwrite_testing:
# Initialize results container
eval_cont = dcase_util.containers.MetaDataContainer(
filename=eval_predictions_filename)
# Load model if not yet loaded
if not keras_model_first_pass:
keras_model_first_pass = keras.models.load_model(
fold1_model_filename)
# Loop through all test files from the current cross-validation fold
for item in db.test(fold=fold):
# Get feature filename
feature_filename = dcase_util.utils.Path(
path=item.filename).modify(path_base=param.get_path(
'path.application.feature_extractor'),
filename_extension='.cpickle')
features = feature_processing_chain.process(
filename=feature_filename)
input_data = features.data.reshape(
features.shape[:-1]).T # (500, 64)
input_data = input_data.reshape(
(1, ) + input_data.shape) # (1, 500, 64)
# Get network output
probabilities = keras_model_first_pass.predict(x=input_data)
# Binarization of the network output
frame_decisions = dcase_util.data.ProbabilityEncoder(
).binarization(probabilities=probabilities,
binarization_type='class_threshold',
threshold=thresholds,
time_axis=0)
estimated_tags = dcase_util.data.DecisionEncoder(
label_list=db.tags()).many_hot(
frame_decisions=frame_decisions, time_axis=0)
# Store result into results container
eval_cont.append({
'filename': item.filename,
'tags': estimated_tags[0]
})
# Save results container
eval_cont.save()
log.foot()
# =====================================================================
# Learning stage 2nd pass, learn from weak and unlabel_in_domain annotated data
# =====================================================================
fold = 2
log.line(data='Fold [{fold}]'.format(fold=fold), indent=2)
# Get model filename
fold2_model_filename = os.path.join(
param.get_path('path.application.learner'),
'model_fold_{fold}.h5'.format(fold=fold))
if not os.path.isfile(fold2_model_filename) or overwrite_learning:
model_parameter_constants = {
'NB_CLASSES':
db.tag_count(),
'INPUT_FREQUENCIES':
param.get_path('feature_extractor.parameters.mel.n_mels'),
'INPUT_SEQUENCE_LENGTH':
param.get_path('feature_sequencer.sequence_length'),
}
model_parameter_constants.update(
param.get_path('learner.parameters.model.constants', {}))
keras_model_second_pass = dcase_util.keras.create_sequential_model(
model_parameter_list=param.get_path(
'learner.parameters.model.second_pass.config'),
constants=model_parameter_constants)
keras_model_second_pass.summary(print_fn=log.line)
# Create optimizer object
param.set_path(path='learner.parameters.compile.optimizer',
new_value=dcase_util.keras.create_optimizer(
class_name=param.get_path(
'learner.parameters.optimizer.class_name'),
config=param.get_path(
'learner.parameters.optimizer.config')))
# Compile model
keras_model_second_pass.compile(
**param.get_path('learner.parameters.compile'))
# Get annotations from the 1st pass model
fold1_results_filename = os.path.join(
param.get_path('path.application.recognizer'),
'pred_weak_fold_{fold}.txt'.format(fold=1))
# Load annotations
predictions_first_pass = dcase_util.containers.MetaDataContainer(
filename=fold1_results_filename).load()
# Split the dataset into train and validation. If "weak" is provided, files from weak.csv are used to
# validate the model. Else, give a percentage which will be used
if param.get_path(
'learner.parameters.model.second_pass.validation_amount'
) == "weak":
training_files = predictions_first_pass.unique_files
training_items = predictions_first_pass
validation_files = db.train(fold=1).unique_files
validation_items = db.train(fold=1)
else:
# Get validation files
training_files, validation_files = db.validation_split(
fold=fold,
split_type='random',
validation_amount=param.get_path(
'learner.parameters.model.second_pass.validation_amount'
),
verbose=False)
training_fold2 = predictions_first_pass + db.train(fold=1)
training_items = training_fold2.filter(
file_list=training_files)
validation_items = training_fold2.filter(
file_list=validation_files)
processing_interval = param.get_path(
'learner.parameters.callbacks.ProgressLoggerCallback.processing_interval'
)
epochs = param.get_path(
'learner.parameters.model.second_pass.fit.epochs')
batch_size = param.get_path(
'learner.parameters.model.second_pass.fit.batch_size')
shuffle = param.get_path(
'learner.parameters.model.second_pass.fit.shuffle')
# Create generators, which convert filename and item into arrays batch_X, batch_y in right formats
training_generator = data_generator(
training_items,
param.get_path('path.application.feature_extractor'),
many_hot_encoder,
feature_processing_chain,
batch_size=batch_size,
shuffle=shuffle,
mode="strong")
validation_generator = data_generator(
validation_items,
param.get_path('path.application.feature_extractor'),
many_hot_encoder,
feature_processing_chain,
batch_size=batch_size,
shuffle=False,
mode="strong")
# Initialize callbacks used during training
callback_list = [
dcase_util.keras.ProgressLoggerCallback(
epochs=param.get_path(
'learner.parameters.model.second_pass.fit.epochs'),
metric=param.get_path(
'learner.parameters.compile.metrics')[0],
loss=param.get_path('learner.parameters.compile.loss'),
output_type='logging',
**param.get_path(
'learner.parameters.callbacks.ProgressLoggerCallback'))
]
if param.get_path('learner.parameters.callbacks.StopperCallback'):
callback_list.append(
dcase_util.keras.StopperCallback(
epochs=param.get_path(
'learner.parameters.model.second_pass.fit.epochs'),
**param.get_path(
'learner.parameters.callbacks.StopperCallback')))
if param.get_path('learner.parameters.callbacks.StasherCallback'):
callback_list.append(
dcase_util.keras.StasherCallback(
epochs=param.get_path(
'learner.parameters.model.second_pass.fit.epochs'),
**param.get_path(
'learner.parameters.callbacks.StasherCallback')))
for epoch_start in range(0, epochs, processing_interval):
epoch_end = epoch_start + processing_interval
# Make sure we have only specified amount of epochs
if epoch_end > epochs:
epoch_end = epochs
# Train keras_model_second_pass
keras_model_second_pass.fit_generator(
generator=training_generator,
steps_per_epoch=len(training_files) // batch_size,
validation_data=validation_generator,
validation_steps=len(validation_files) // batch_size,
callbacks=callback_list,
verbose=0,
initial_epoch=epoch_start,
epochs=epoch_end)
# Calculate external metrics, f_measure of the current epoch
val_macro_f_measure = get_f_measure_by_class(
keras_model_second_pass,
db.tag_count(),
validation_generator,
len(validation_files) // batch_size,
)
val_macro_f_measure = val_macro_f_measure.mean()
tra_macro_f_measure = get_f_measure_by_class(
keras_model_second_pass,
db.tag_count(),
training_generator,
len(training_files) // batch_size,
)
tra_macro_f_measure = tra_macro_f_measure.mean()
# Inject external metric values to the callbacks
for callback in callback_list:
if hasattr(callback, 'set_external_metric_value'):
callback.set_external_metric_value(
metric_label='val_macro_f_measure',
metric_value=val_macro_f_measure)
callback.set_external_metric_value(
metric_label='tra_macro_f_measure',
metric_value=tra_macro_f_measure)
# Manually update callbacks
for callback in callback_list:
if hasattr(callback, 'update'):
callback.update()
# Check we need to stop training
stop_training = False
for callback in callback_list:
if hasattr(callback, 'stop'):
if callback.stop():
log.line("Early stropping")
stop_training = True
if stop_training:
# Stop the training loop
break
# Fetch best model
for callback in callback_list:
if isinstance(callback, dcase_util.keras.StasherCallback):
callback.log()
best_weights = callback.get_best()['weights']
if best_weights:
keras_model_second_pass.set_weights(best_weights)
break
# Save trained model
keras_model_second_pass.save(fold2_model_filename)
log.foot()
# =====================================================================
# Testing stage, get strong annotations
# =====================================================================
fold = 2
if param.get_path('flow.testing'):
log.section_header('Testing')
# Get results filename
test_predictions_filename = os.path.join(
param.get_path('path.application.recognizer'),
'test_fold_{fold}.csv'.format(fold=2))
# Get model filename
fold2_model_filename = os.path.join(
param.get_path('path.application.learner'),
'res_fold_{fold}.h5'.format(fold=fold))
if not os.path.isfile(test_predictions_filename) or overwrite_testing:
# Load model if not yet loaded
if not keras_model_second_pass:
keras_model_second_pass = keras.models.load_model(
"/home/nturpault/model_fold_2.h5")
# Initialize results container
test_meta = dcase_util.containers.MetaDataContainer(
filename=test_predictions_filename)
# Loop through all test files from the current cross-validation fold
for item in db.test(fold=fold):
# Get feature filename
feature_filename = dcase_util.utils.Path(
path=item.filename).modify(path_base=param.get_path(
'path.application.feature_extractor'),
filename_extension='.cpickle')
# Get features array
features = feature_processing_chain.process(
filename=feature_filename)
input_data = features.data.reshape(
features.shape[:-1]).T # (500, 64)
# Create a batch with only one file
input_data = input_data.reshape(
(1, ) + input_data.shape) # (1, 500, 64)
# Get network output for strong data
probabilities = keras_model_second_pass.predict(input_data)
# only one file in the batch
probabilities = probabilities[0]
if param.get_path('recognizer.frame_binarization.enable'):
# Binarization of the network output
frame_decisions = dcase_util.data.ProbabilityEncoder(
).binarization(
probabilities=probabilities,
binarization_type=param.get_path(
'recognizer.frame_binarization.binarization_type'),
threshold=param.get_path(
'recognizer.frame_binarization.threshold'),
time_axis=0)
else:
frame_decisions = dcase_util.data.ProbabilityEncoder(
).binarization(probabilities=probabilities,
binarization_type="global_threshold",
threshold=0.5,
time_axis=0)
decision_encoder = dcase_util.data.DecisionEncoder(
label_list=db.tags())
if param.get_path('recognizer.process_activity.enable'):
frame_decisions = decision_encoder.process_activity(
frame_decisions,
window_length=param.get_path(
'recognizer.process_activity.window_length'),
time_axis=0)
for i, label in enumerate(db.tags()):
# given a list of ones, give the onset and offset in frames
estimated_events = decision_encoder.find_contiguous_regions(
activity_array=frame_decisions[:, i])
for [onset, offset] in estimated_events:
hop_length_seconds = param.get_path(
'feature_extractor.hop_length_seconds')
# Store result into results container, convert frames to seconds
test_meta.append({
'filename': item.filename,
'event_label': label,
'onset': onset * hop_length_seconds,
'offset': offset * hop_length_seconds
})
# Save results container
test_meta.save()
stats_filename = os.path.join(
param.get_path('path.application.recognizer'), 'test_results.txt')
if not os.path.isfile(stats_filename) or overwrite_testing:
# test data used to evaluate the system
reference_event_list = db.test(fold=fold)
# Evaluation done on dataframes
reference_df = pandas.DataFrame(reference_event_list)
# predictions done during the step test before
eval_df = pandas.read_csv(test_predictions_filename,
sep="\t",
header=0)
# Calculate the metric
event_based_metric = event_based_evaluation_df(
reference_df, eval_df)
with open(stats_filename, "w") as stats_file:
stats_file.write(event_based_metric.__str__())
log.line(event_based_metric.__str__(), indent=4)
log.foot()
# =====================================================================
# Evaluation stage, predict evaluation labels
# =====================================================================
if param.get_path('flow.evaluation'):
log.section_header('Evaluation')
# Get results filename
eval_predictions_filename = os.path.join(
param.get_path('path.application.recognizer'), 'eval_results.csv')
# Get model filename
fold2_model_filename = os.path.join(
param.get_path('path.application.learner'),
'model_fold_{fold}.h5'.format(fold=2))
if not os.path.isfile(
eval_predictions_filename) or overwrite_evaluation:
# Load model if not yet loaded
if not keras_model_second_pass:
keras_model_second_pass = keras.models.load_model(
fold2_model_filename)
# Initialize results container
eval_meta = dcase_util.containers.MetaDataContainer(
filename=eval_predictions_filename)
# Loop through all test files from the current cross-validation fold
for item in db.eval(fold=2):
# Get feature filename
feature_filename = dcase_util.utils.Path(
path=item.filename).modify(path_base=param.get_path(
'path.application.feature_extractor'),
filename_extension='.cpickle')
# Get features array
features = feature_processing_chain.process(
filename=feature_filename)
input_data = features.data.reshape(
features.shape[:-1]).T # (500, 64)
# Create a batch with only one file
input_data = input_data.reshape(
(1, ) + input_data.shape) # (1, 500, 64)
# Get network output for strong data
probabilities = keras_model_second_pass.predict(input_data)
# only one file in the batch
probabilities = probabilities[0]
if param.get_path('recognizer.frame_binarization.enable'):
# Binarization of the network output
frame_decisions = dcase_util.data.ProbabilityEncoder(
).binarization(
probabilities=probabilities,
binarization_type=param.get_path(
'recognizer.frame_binarization.binarization_type'),
threshold=param.get_path(
'recognizer.frame_binarization.threshold'),
time_axis=0)
else:
frame_decisions = dcase_util.data.ProbabilityEncoder(
).binarization(probabilities=probabilities,
binarization_type="global_threshold",
threshold=0.5,
time_axis=0)
decision_encoder = dcase_util.data.DecisionEncoder(
label_list=db.tags())
if param.get_path('recognizer.process_activity.enable'):
frame_decisions = decision_encoder.process_activity(
frame_decisions,
window_length=param.get_path(
'recognizer.process_activity.window_length'),
time_axis=0)
for i, label in enumerate(db.tags()):
# given a list of ones, give the onset and offset in frames
estimated_events = decision_encoder.find_contiguous_regions(
activity_array=frame_decisions[:, i])
for [onset, offset] in estimated_events:
hop_length_seconds = param.get_path(
'feature_extractor.hop_length_seconds')
# Store result into results container, convert frames to seconds
eval_meta.append({
'filename': item.filename,
'event_label': label,
'onset': onset * hop_length_seconds,
'offset': offset * hop_length_seconds
})
# Save results container
eval_meta.save()
log.line("Find the eval predictions in : " + eval_predictions_filename)
stats_filename = os.path.join(
param.get_path('path.application.recognizer'), 'eval_results.txt')
if not os.path.isfile(stats_filename) or overwrite_evaluation:
# get the two dataframes with events and evaluate the system
reference_event_list = db.eval(fold=2)
reference_df = pandas.DataFrame(reference_event_list)
eval_df = pandas.read_csv(eval_predictions_filename,
sep="\t",
header=0)
# Calculate the metric
event_based_metric = event_based_evaluation_df(
reference_df, eval_df)
with open(stats_filename, "w") as stats_file:
stats_file.write(event_based_metric.__str__())
log.line(event_based_metric.__str__(), indent=4)
log.foot()
loss_bce = criterion_bce(weak_out, pred_labels)
loss_bce.backward()
loss_bce.append(loss_bce.item())
optimizer.step()
loss_bce = np.mean(loss_bce)
print('[%d / %d, %5d] loss: %.3f' %
(epoch_ + 1, n_epochs, cnt + 1, loss_bce))
return loss_bce, model
if not os.path.exists(model_path_sup1) or cfg.recompute_classif:
for epoch_ in range(n_epochs):
start = time.time()
loss_mean_bce, model = train_loop(train_load, model)
model.eval()
macro_f_measure_train = get_f_measure_by_class(
model, len(classes), train_set_emb, max=args.single_label)
macro_f_measure_val = get_f_measure_by_class(model,
len(classes),
valid_set_val,
max=args.single_label)
mean_macro_f_measure = np.mean(macro_f_measure_val)
macro_f_measure_test = get_f_measure_by_class(
model, len(classes), test_set_val, max=args.single_label)
model.train()
print("Time to train an epoch: {}".format(time.time() - start))
# print statistics
results = {
"train_loss":
epoch,
ema_model=crnn_ema,
weak_mask=weak_mask,
strong_mask=strong_mask)
crnn = crnn.eval()
LOG.info("\n ### Validation Metrics ### \n")
# predictions = get_predictions(crnn, validation_data, many_hot_encoder.decode_strong,
# save_predictions=None)
# pdf = predictions.copy()
# pdf.filename = pdf.filename.str.replace('.npy', '.wav')
# valid_events_metric = compute_strong_metrics(pdf, valid_synth_df, pooling_time_ratio)
# LOG.info("\n ### Valid weak metric ### \n")
weak_metric = get_f_measure_by_class(
crnn, len(cfg.classes),
DataLoader(validation_data, batch_size=cfg.batch_size))
LOG.info("Weak F1-score per class: \n {}".format(
pd.DataFrame(weak_metric * 100, many_hot_encoder.labels)))
LOG.info("Weak F1-score macro averaged: {}".format(
np.mean(weak_metric)))
state['model']['state_dict'] = crnn.state_dict()
state['model_ema']['state_dict'] = crnn_ema.state_dict()
state['optimizer']['state_dict'] = optimizer.state_dict()
state['epoch'] = epoch
# state['valid_metric'] = valid_events_metric.results()
if cfg.checkpoint_epochs is not None and (
epoch + 1) % cfg.checkpoint_epochs == 0:
model_fname = os.path.join(saved_model_dir, '_epoch_' + str(epoch))