def test_model(state,
reference_tsv_path,
reduced_number_of_data=None,
strore_predicitions_fname=None):
dataset = DatasetDcase2019Task4(os.path.join(cfg.workspace),
base_feature_dir=os.path.join(
cfg.workspace, "dataset", "features"),
save_log_feature=False)
crnn_kwargs = state["model"]["kwargs"]
crnn = CRNN(**crnn_kwargs)
crnn.load(parameters=state["model"]["state_dict"])
LOG.info("Model loaded at epoch: {}".format(state["epoch"]))
pooling_time_ratio = state["pooling_time_ratio"]
crnn.load(parameters=state["model"]["state_dict"])
scaler = Scaler()
scaler.load_state_dict(state["scaler"])
classes = cfg.classes
many_hot_encoder = ManyHotEncoder.load_state_dict(
state["many_hot_encoder"])
crnn = crnn.eval()
[crnn] = to_cuda_if_available([crnn])
transforms_valid = get_transforms(cfg.max_frames, scaler=scaler)
LOG.info(reference_tsv_path)
df = dataset.initialize_and_get_df(reference_tsv_path,
reduced_number_of_data)
strong_dataload = DataLoadDf(df,
dataset.get_feature_file,
many_hot_encoder.encode_strong_df,
transform=transforms_valid)
predictions = get_predictions(crnn,
strong_dataload,
many_hot_encoder.decode_strong,
pooling_time_ratio,
save_predictions=strore_predicitions_fname)
compute_strong_metrics(predictions, df)
weak_dataload = DataLoadDf(df,
dataset.get_feature_file,
many_hot_encoder.encode_weak,
transform=transforms_valid)
weak_metric = get_f_measure_by_class(
crnn, len(classes), DataLoader(weak_dataload,
batch_size=cfg.batch_size))
LOG.info("Weak F1-score per class: \n {}".format(
pd.DataFrame(weak_metric * 100, many_hot_encoder.labels)))
LOG.info("Weak F1-score macro averaged: {}".format(np.mean(weak_metric)))
loss_value, meters = train(training_loader,
crnn,
optim,
epoch,
ema_model=crnn_ema,
mask_weak=weak_mask,
mask_strong=strong_mask,
adjust_lr=cfg.adjust_lr)
# Validation
crnn = crnn.eval()
logger.info("\n ### Valid synthetic metric ### \n")
predictions = get_predictions(crnn,
valid_synth_loader,
many_hot_encoder.decode_strong,
pooling_time_ratio,
median_window=median_window,
save_predictions=None)
# Validation with synthetic data (dropping feature_filename for psds)
valid_synth = dfs["valid_synthetic"].drop("feature_filename", axis=1)
valid_synth_f1, psds_m_f1 = compute_metrics(predictions, valid_synth,
durations_synth)
# ---------------
# Save the trainable PCEN parameters (if any)
if crnn.trainable_pcen is not None:
pcen_parameters.append({
key: torch.tensor(value)
for key, value in crnn.trainable_pcen.state_dict().items()
})
# print("PCEN PARAMS:", pcen_parameters)
def test_model(state, reduced_number_of_data, strore_predicitions_fname=None):
crnn_kwargs = state["model"]["kwargs"]
crnn = CRNN(**crnn_kwargs)
crnn.load(parameters=state["model"]["state_dict"])
LOG.info("Model loaded at epoch: {}".format(state["epoch"]))
pooling_time_ratio = state["pooling_time_ratio"]
crnn.load(parameters=state["model"]["state_dict"])
scaler = Scaler()
scaler.load_state_dict(state["scaler"])
classes = cfg.classes
many_hot_encoder = ManyHotEncoder.load_state_dict(
state["many_hot_encoder"])
# ##############
# Validation
# ##############
crnn = crnn.eval()
[crnn] = to_cuda_if_available([crnn])
transforms_valid = get_transforms(cfg.max_frames, scaler=scaler)
# # 2018
# LOG.info("Eval 2018")
# eval_2018_df = dataset.initialize_and_get_df(cfg.eval2018, reduced_number_of_data)
# # Strong
# eval_2018_strong = DataLoadDf(eval_2018_df, dataset.get_feature_file, many_hot_encoder.encode_strong_df,
# transform=transforms_valid)
# predictions = get_predictions(crnn, eval_2018_strong, many_hot_encoder.decode_strong)
# compute_strong_metrics(predictions, eval_2018_df, pooling_time_ratio)
# # Weak
# eval_2018_weak = DataLoadDf(eval_2018_df, dataset.get_feature_file, many_hot_encoder.encode_weak,
# transform=transforms_valid)
# weak_metric = get_f_measure_by_class(crnn, len(classes), DataLoader(eval_2018_weak, batch_size=cfg.batch_size))
# LOG.info("Weak F1-score per class: \n {}".format(pd.DataFrame(weak_metric * 100, many_hot_encoder.labels)))
# LOG.info("Weak F1-score macro averaged: {}".format(np.mean(weak_metric)))
# Validation 2019
# LOG.info("Validation 2019 (original code)")
# b_dataset = B_DatasetDcase2019Task4(cfg.workspace,
# base_feature_dir=os.path.join(cfg.workspace, 'dataset', 'features'),
# save_log_feature=False)
# b_validation_df = b_dataset.initialize_and_get_df(cfg.validation, reduced_number_of_data)
# b_validation_df.to_csv('old.csv')
# b_validation_strong = B_DataLoadDf(b_validation_df,
# b_dataset.get_feature_file, many_hot_encoder.encode_strong_df,
# transform=transforms_valid)
# predictions2 = get_predictions(crnn, b_validation_strong, many_hot_encoder.decode_strong,
# save_predictions=strore_predicitions_fname)
# compute_strong_metrics(predictions2, b_validation_df, pooling_time_ratio)
# b_validation_weak = B_DataLoadDf(b_validation_df, b_dataset.get_feature_file, many_hot_encoder.encode_weak,
# transform=transforms_valid)
# weak_metric = get_f_measure_by_class(crnn, len(classes), DataLoader(b_validation_weak, batch_size=cfg.batch_size))
# LOG.info("Weak F1-score per class: \n {}".format(pd.DataFrame(weak_metric * 100, many_hot_encoder.labels)))
# LOG.info("Weak F1-score macro averaged: {}".format(np.mean(weak_metric)))
# ============================================================================================
# ============================================================================================
# ============================================================================================
dataset = DatasetDcase2019Task4(feature_dir=cfg.feature_dir,
local_path=cfg.workspace,
exp_tag=cfg.exp_tag,
save_log_feature=False)
# Validation 2019
LOG.info("Validation 2019")
validation_df = dataset.initialize_and_get_df(cfg.validation,
reduced_number_of_data)
validation_strong = DataLoadDf(validation_df,
dataset.get_feature_file,
many_hot_encoder.encode_strong_df,
transform=transforms_valid)
predictions = get_predictions(crnn,
validation_strong,
many_hot_encoder.decode_strong,
save_predictions=strore_predicitions_fname)
vdf = validation_df.copy()
vdf.filename = vdf.filename.str.replace('.npy', '.wav')
pdf = predictions.copy()
pdf.filename = pdf.filename.str.replace('.npy', '.wav')
compute_strong_metrics(pdf, vdf, pooling_time_ratio)
validation_weak = DataLoadDf(validation_df,
dataset.get_feature_file,
many_hot_encoder.encode_weak,
transform=transforms_valid)
weak_metric = get_f_measure_by_class(
crnn, len(classes),
DataLoader(validation_weak, batch_size=cfg.batch_size))
LOG.info("Weak F1-score per class: \n {}".format(
pd.DataFrame(weak_metric * 100, many_hot_encoder.labels)))
LOG.info("Weak F1-score macro averaged: {}".format(np.mean(weak_metric)))
loss_value = train(training_loader,
crnn,
optim,
epoch,
ema_model=crnn_ema,
mask_weak=weak_mask,
mask_strong=strong_mask,
adjust_lr=cfg.adjust_lr)
# Validation
crnn = crnn.eval()
logger.info("\n ### Valid synthetic metric ### \n")
predictions = get_predictions(crnn,
valid_synth_loader,
many_hot_encoder.decode_strong,
pooling_time_ratio,
median_window=median_window,
save_predictions=None)
# Validation with synthetic data (dropping feature_filename for psds)
valid_synth = dfs["valid_synthetic"].drop("feature_filename", axis=1)
valid_events_metric = compute_sed_eval_metrics(predictions,
valid_synth)
psds_results(predictions, valid_synth, durations_synth)
# Update state
state['model']['state_dict'] = crnn.state_dict()
state['model_ema']['state_dict'] = crnn_ema.state_dict()
state['optimizer']['state_dict'] = optim.state_dict()
state['epoch'] = epoch
state['valid_metric'] = valid_events_metric.results()
# Model
expe_state = torch.load(model_path, map_location="cpu")
dataset = DESED(base_feature_dir=osp.join(cfg.workspace, "dataset",
"features"),
compute_log=False)
gt_df_feat = dataset.initialize_and_get_df(f_args.groundtruth_tsv,
gt_audio_dir,
nb_files=f_args.nb_files)
params = _load_state_vars(expe_state, gt_df_feat, median_window)
# Preds with only one value
single_predictions = get_predictions(
params["model"],
params["dataloader"],
params["many_hot_encoder"].decode_strong,
params["pooling_time_ratio"],
median_window=params["median_window"],
save_predictions=f_args.save_predictions_path)
compute_metrics(single_predictions, groundtruth, durations)
# ##########
# Optional but recommended
# ##########
# Compute psds scores with multiple thresholds (more accurate). n_thresholds could be increased.
n_thresholds = 50
# Example of 5 thresholds: 0.1, 0.3, 0.5, 0.7, 0.9
list_thresholds = np.arange(1 / (n_thresholds * 2), 1, 1 / n_thresholds)
pred_ss_thresh = get_predictions(
params["model"],
params["dataloader"],
eval_2018 = DataLoadDf(eval_2018_df,
dataset.get_feature_file,
many_hot_encoder.encode_strong_df,
transform=transforms_valid)
[crnn] = to_cuda_if_available([crnn])
for epoch in range(cfg.n_epoch):
crnn = crnn.train()
train(training_data, crnn, optimizer, epoch, weak_mask, strong_mask)
crnn = crnn.eval()
LOG.info("Training synthetic metric:")
train_predictions = get_predictions(crnn,
train_synth_data,
many_hot_encoder.decode_strong,
pooling_time_ratio,
save_predictions=None)
train_metric = compute_strong_metrics(train_predictions,
train_synth_df)
if not no_weak:
LOG.info("Training weak metric:")
weak_metric = get_f_measure_by_class(
crnn, len(classes),
DataLoader(train_weak_data, batch_size=cfg.batch_size))
LOG.info("Weak F1-score per class: \n {}".format(
pd.DataFrame(weak_metric * 100, many_hot_encoder.labels)))
LOG.info("Weak F1-score macro averaged: {}".format(
np.mean(weak_metric)))
"Number of files to be used. Useful when testing on small number of files."
)
f_args = parser.parse_args()
# Get variables from f_args
model_path, median_window, gt_audio_dir, groundtruth, durations = get_variables(
f_args)
# Model
expe_state = torch.load(model_path, map_location="cpu")
dataset = DESED(base_feature_dir=osp.join(cfg.workspace, "dataset",
"features"),
compute_log=False)
gt_df_feat = dataset.initialize_and_get_df(f_args.groundtruth_tsv,
gt_audio_dir,
nb_files=f_args.nb_files)
params = _load_state_vars(expe_state, gt_df_feat, median_window)
# Preds with only one value
single_predictions = get_predictions(
params["ps"],
params["ema"],
params["ms"],
params["dataloader"],
params["many_hot_encoder"].decode_strong,
params["pooling_time_ratio"],
median_window=params["median_window"],
save_predictions=f_args.save_predictions_path)
compute_metrics(single_predictions, groundtruth, durations)
save_best_cb = SaveBest("sup")
# ##############
# Train
# ##############
for epoch in range(cfg.n_epoch):
crnn = crnn.train()
crnn_ema = crnn_ema.train()
[crnn, crnn_ema] = to_cuda_if_available([crnn, crnn_ema])
train(training_data, crnn, optimizer, epoch, ema_model=crnn_ema, weak_mask=weak_mask, strong_mask=strong_mask)
crnn = crnn.eval()
LOG.info("\n ### Valid synthetic metric ### \n")
predictions = get_predictions(crnn, valid_synth_data, many_hot_encoder.decode_strong,
save_predictions=None)
valid_events_metric = compute_strong_metrics(predictions, valid_synth_df, pooling_time_ratio)
LOG.info("\n ### Valid weak metric ### \n")
weak_metric = get_f_measure_by_class(crnn, len(classes),
DataLoader(valid_weak_data, batch_size=cfg.batch_size))
LOG.info("Weak F1-score per class: \n {}".format(pd.DataFrame(weak_metric * 100, many_hot_encoder.labels)))
LOG.info("Weak F1-score macro averaged: {}".format(np.mean(weak_metric)))
state['model']['state_dict'] = crnn.state_dict()
state['model_ema']['state_dict'] = crnn_ema.state_dict()
state['optimizer']['state_dict'] = optimizer.state_dict()
state['epoch'] = epoch
state['valid_metric'] = valid_events_metric.results()
if cfg.checkpoint_epochs is not None and (epoch + 1) % cfg.checkpoint_epochs == 0:
results = pd.DataFrame(columns=["loss", "valid_synth_f1", "weak_metric", "global_valid"])
for epoch in range(cfg.n_epoch):
MS_model.train()
MS_ema.train()
PS_model.train()
MS_model, MS_ema, PS_model = to_cuda_if_available(MS_model, MS_ema, PS_model)
loss_value = train(training_loader, MS_model, PS_model, ms_optim, ps_optim, epoch,
ema_model=MS_ema, mask_weak=weak_mask, mask_strong=strong_mask, adjust_lr=cfg.adjust_lr)
# Validation
ema = MS_ema.eval()
MS_m = MS_model.eval()
PS_m = PS_model.eval()
logger.info("\n ### Valid synthetic metric ### \n")
predictions = get_predictions(PS_m, ema, MS_m, valid_synth_loader, many_hot_encoder.decode_strong, pooling_time_ratio,
median_window=median_window, save_predictions=None)
# Validation with synthetic data (dropping feature_filename for psds)
valid_synth = dfs["validation"].drop("feature_filename", axis=1)
valid_synth_f1, psds_m_f1 = compute_metrics(predictions, valid_synth, durations_validation)
# Update state
state['MS']['state_dict'] = MS_model.state_dict()
state['PS']['state_dict'] = PS_model.state_dict()
state['model_ema']['state_dict'] = MS_ema.state_dict()
state['ms_optimizer']['state_dict'] = ms_optim.state_dict()
state['ps_optimizer']['state_dict'] = ps_optim.state_dict()
state['epoch'] = epoch
state['valid_metric'] = valid_synth_f1
state['valid_f1_psds'] = psds_m_f1
# Callbacks