def test_model(state,
reference_tsv_path,
reduced_number_of_data=None,
strore_predicitions_fname=None):
dataset = DatasetDcase2019Task4(os.path.join(cfg.workspace),
base_feature_dir=os.path.join(
cfg.workspace, "dataset", "features"),
save_log_feature=False)
crnn_kwargs = state["model"]["kwargs"]
crnn = CRNN(**crnn_kwargs)
crnn.load(parameters=state["model"]["state_dict"])
LOG.info("Model loaded at epoch: {}".format(state["epoch"]))
pooling_time_ratio = state["pooling_time_ratio"]
crnn.load(parameters=state["model"]["state_dict"])
scaler = Scaler()
scaler.load_state_dict(state["scaler"])
classes = cfg.classes
many_hot_encoder = ManyHotEncoder.load_state_dict(
state["many_hot_encoder"])
crnn = crnn.eval()
[crnn] = to_cuda_if_available([crnn])
transforms_valid = get_transforms(cfg.max_frames, scaler=scaler)
LOG.info(reference_tsv_path)
df = dataset.initialize_and_get_df(reference_tsv_path,
reduced_number_of_data)
strong_dataload = DataLoadDf(df,
dataset.get_feature_file,
many_hot_encoder.encode_strong_df,
transform=transforms_valid)
predictions = get_predictions(crnn,
strong_dataload,
many_hot_encoder.decode_strong,
pooling_time_ratio,
save_predictions=strore_predicitions_fname)
compute_strong_metrics(predictions, df)
weak_dataload = DataLoadDf(df,
dataset.get_feature_file,
many_hot_encoder.encode_weak,
transform=transforms_valid)
weak_metric = get_f_measure_by_class(
crnn, len(classes), DataLoader(weak_dataload,
batch_size=cfg.batch_size))
LOG.info("Weak F1-score per class: \n {}".format(
pd.DataFrame(weak_metric * 100, many_hot_encoder.labels)))
LOG.info("Weak F1-score macro averaged: {}".format(np.mean(weak_metric)))
def test_model(state, reduced_number_of_data, strore_predicitions_fname=None):
crnn_kwargs = state["model"]["kwargs"]
crnn = CRNN(**crnn_kwargs)
crnn.load(parameters=state["model"]["state_dict"])
LOG.info("Model loaded at epoch: {}".format(state["epoch"]))
pooling_time_ratio = state["pooling_time_ratio"]
crnn.load(parameters=state["model"]["state_dict"])
scaler = Scaler()
scaler.load_state_dict(state["scaler"])
classes = cfg.classes
many_hot_encoder = ManyHotEncoder.load_state_dict(
state["many_hot_encoder"])
# ##############
# Validation
# ##############
crnn = crnn.eval()
[crnn] = to_cuda_if_available([crnn])
transforms_valid = get_transforms(cfg.max_frames, scaler=scaler)
# # 2018
# LOG.info("Eval 2018")
# eval_2018_df = dataset.initialize_and_get_df(cfg.eval2018, reduced_number_of_data)
# # Strong
# eval_2018_strong = DataLoadDf(eval_2018_df, dataset.get_feature_file, many_hot_encoder.encode_strong_df,
# transform=transforms_valid)
# predictions = get_predictions(crnn, eval_2018_strong, many_hot_encoder.decode_strong)
# compute_strong_metrics(predictions, eval_2018_df, pooling_time_ratio)
# # Weak
# eval_2018_weak = DataLoadDf(eval_2018_df, dataset.get_feature_file, many_hot_encoder.encode_weak,
# transform=transforms_valid)
# weak_metric = get_f_measure_by_class(crnn, len(classes), DataLoader(eval_2018_weak, batch_size=cfg.batch_size))
# LOG.info("Weak F1-score per class: \n {}".format(pd.DataFrame(weak_metric * 100, many_hot_encoder.labels)))
# LOG.info("Weak F1-score macro averaged: {}".format(np.mean(weak_metric)))
# Validation 2019
# LOG.info("Validation 2019 (original code)")
# b_dataset = B_DatasetDcase2019Task4(cfg.workspace,
# base_feature_dir=os.path.join(cfg.workspace, 'dataset', 'features'),
# save_log_feature=False)
# b_validation_df = b_dataset.initialize_and_get_df(cfg.validation, reduced_number_of_data)
# b_validation_df.to_csv('old.csv')
# b_validation_strong = B_DataLoadDf(b_validation_df,
# b_dataset.get_feature_file, many_hot_encoder.encode_strong_df,
# transform=transforms_valid)
# predictions2 = get_predictions(crnn, b_validation_strong, many_hot_encoder.decode_strong,
# save_predictions=strore_predicitions_fname)
# compute_strong_metrics(predictions2, b_validation_df, pooling_time_ratio)
# b_validation_weak = B_DataLoadDf(b_validation_df, b_dataset.get_feature_file, many_hot_encoder.encode_weak,
# transform=transforms_valid)
# weak_metric = get_f_measure_by_class(crnn, len(classes), DataLoader(b_validation_weak, batch_size=cfg.batch_size))
# LOG.info("Weak F1-score per class: \n {}".format(pd.DataFrame(weak_metric * 100, many_hot_encoder.labels)))
# LOG.info("Weak F1-score macro averaged: {}".format(np.mean(weak_metric)))
# ============================================================================================
# ============================================================================================
# ============================================================================================
dataset = DatasetDcase2019Task4(feature_dir=cfg.feature_dir,
local_path=cfg.workspace,
exp_tag=cfg.exp_tag,
save_log_feature=False)
# Validation 2019
LOG.info("Validation 2019")
validation_df = dataset.initialize_and_get_df(cfg.validation,
reduced_number_of_data)
validation_strong = DataLoadDf(validation_df,
dataset.get_feature_file,
many_hot_encoder.encode_strong_df,
transform=transforms_valid)
predictions = get_predictions(crnn,
validation_strong,
many_hot_encoder.decode_strong,
save_predictions=strore_predicitions_fname)
vdf = validation_df.copy()
vdf.filename = vdf.filename.str.replace('.npy', '.wav')
pdf = predictions.copy()
pdf.filename = pdf.filename.str.replace('.npy', '.wav')
compute_strong_metrics(pdf, vdf, pooling_time_ratio)
validation_weak = DataLoadDf(validation_df,
dataset.get_feature_file,
many_hot_encoder.encode_weak,
transform=transforms_valid)
weak_metric = get_f_measure_by_class(
crnn, len(classes),
DataLoader(validation_weak, batch_size=cfg.batch_size))
LOG.info("Weak F1-score per class: \n {}".format(
pd.DataFrame(weak_metric * 100, many_hot_encoder.labels)))
LOG.info("Weak F1-score macro averaged: {}".format(np.mean(weak_metric)))
transform=transforms_valid)
[crnn] = to_cuda_if_available([crnn])
for epoch in range(cfg.n_epoch):
crnn = crnn.train()
train(training_data, crnn, optimizer, epoch, weak_mask, strong_mask)
crnn = crnn.eval()
LOG.info("Training synthetic metric:")
train_predictions = get_predictions(crnn,
train_synth_data,
many_hot_encoder.decode_strong,
pooling_time_ratio,
save_predictions=None)
train_metric = compute_strong_metrics(train_predictions,
train_synth_df)
if not no_weak:
LOG.info("Training weak metric:")
weak_metric = get_f_measure_by_class(
crnn, len(classes),
DataLoader(train_weak_data, batch_size=cfg.batch_size))
LOG.info("Weak F1-score per class: \n {}".format(
pd.DataFrame(weak_metric * 100, many_hot_encoder.labels)))
LOG.info("Weak F1-score macro averaged: {}".format(
np.mean(weak_metric)))
LOG.info("Valid weak metric:")
weak_metric = get_f_measure_by_class(
crnn, len(classes),
DataLoader(valid_weak_data, batch_size=cfg.batch_size))
# ##############
# Train
# ##############
for epoch in range(cfg.n_epoch):
crnn = crnn.train()
crnn_ema = crnn_ema.train()
[crnn, crnn_ema] = to_cuda_if_available([crnn, crnn_ema])
train(training_data, crnn, optimizer, epoch, ema_model=crnn_ema, weak_mask=weak_mask, strong_mask=strong_mask)
crnn = crnn.eval()
LOG.info("\n ### Valid synthetic metric ### \n")
predictions = get_predictions(crnn, valid_synth_data, many_hot_encoder.decode_strong,
save_predictions=None)
valid_events_metric = compute_strong_metrics(predictions, valid_synth_df, pooling_time_ratio)
LOG.info("\n ### Valid weak metric ### \n")
weak_metric = get_f_measure_by_class(crnn, len(classes),
DataLoader(valid_weak_data, batch_size=cfg.batch_size))
LOG.info("Weak F1-score per class: \n {}".format(pd.DataFrame(weak_metric * 100, many_hot_encoder.labels)))
LOG.info("Weak F1-score macro averaged: {}".format(np.mean(weak_metric)))
state['model']['state_dict'] = crnn.state_dict()
state['model_ema']['state_dict'] = crnn_ema.state_dict()
state['optimizer']['state_dict'] = optimizer.state_dict()
state['epoch'] = epoch
state['valid_metric'] = valid_events_metric.results()
if cfg.checkpoint_epochs is not None and (epoch + 1) % cfg.checkpoint_epochs == 0:
model_fname = os.path.join(saved_model_dir, "baseline_epoch_" + str(epoch))
# Eval 2018
eval_2018_df = dataset.initialize_and_get_df(cfg.eval2018, reduced_number_of_data)
eval_2018 = DataLoadDf(eval_2018_df, dataset.get_feature_file, many_hot_encoder.encode_strong_df,
transform=transforms_valid)
[crnn] = to_cuda_if_available([crnn])
for epoch in range(cfg.n_epoch):
crnn = crnn.train()
train(training_data, crnn, optimizer, epoch, weak_mask, strong_mask)
crnn = crnn.eval()
LOG.info("Training synthetic metric:")
train_predictions = get_predictions(crnn, train_synth_data, many_hot_encoder.decode_strong,
save_predictions=None)
train_metric = compute_strong_metrics(train_predictions, train_synth_df, pooling_time_ratio)
if not no_weak:
LOG.info("Training weak metric:")
weak_metric = get_f_measure_by_class(crnn, len(classes),
DataLoader(train_weak_data, batch_size=cfg.batch_size))
LOG.info("Weak F1-score per class: \n {}".format(pd.DataFrame(weak_metric * 100, many_hot_encoder.labels)))
LOG.info("Weak F1-score macro averaged: {}".format(np.mean(weak_metric)))
LOG.info("Valid weak metric:")
weak_metric = get_f_measure_by_class(crnn, len(classes),
DataLoader(valid_weak_data, batch_size=cfg.batch_size))
LOG.info(
"Weak F1-score per class: \n {}".format(pd.DataFrame(weak_metric * 100, many_hot_encoder.labels)))
LOG.info("Weak F1-score macro averaged: {}".format(np.mean(weak_metric)))
train(training_data,
crnn,
optimizer,
epoch,
ema_model=crnn_ema,
weak_mask=weak_mask,
strong_mask=strong_mask)
crnn = crnn.eval()
LOG.info("\n ### Valid synthetic metric ### \n")
predictions = get_predictions(crnn,
valid_synth_data,
many_hot_encoder.decode_strong,
pooling_time_ratio,
save_predictions=None)
valid_events_metric = compute_strong_metrics(predictions,
valid_synth_df)
LOG.info("\n ### Valid weak metric ### \n")
weak_metric = get_f_measure_by_class(
crnn, len(classes),
DataLoader(valid_weak_data, batch_size=cfg.batch_size))
LOG.info("Weak F1-score per class: \n {}".format(
pd.DataFrame(weak_metric * 100, many_hot_encoder.labels)))
LOG.info("Weak F1-score macro averaged: {}".format(
np.mean(weak_metric)))
state['model']['state_dict'] = crnn.state_dict()
state['model_ema']['state_dict'] = crnn_ema.state_dict()
state['optimizer']['state_dict'] = optimizer.state_dict()
state['epoch'] = epoch