def loop_batches(samples, model_triplet, semi_hard_input=None, semi_hard_embed=None, i=0):
inputs, inputs_pos, inputs_neg, pred_labels = samples
inputs, inputs_pos = to_cuda_if_available(inputs, inputs_pos)
if i < 2:
LOG.debug("input shape: {}".format(inputs.shape))
if semi_hard_input is not None or semi_hard_embed is not None:
assert semi_hard_input is not None, "semi_hard_input and semi_hard_embed should be defined"
assert semi_hard_embed is not None, "semi_hard_input and semi_hard_embed should be defined"
model_triplet.eval()
embed = get_embeddings_numpy(inputs, model_triplet)
embed_pos = get_embeddings_numpy(inputs_pos, model_triplet)
label_mask = (pred_labels.numpy() == -1).all(-1)
semi_hard_mask = np.isnan(inputs_neg.detach().numpy()).reshape(inputs_neg.shape[0], -1).all(-1)
mask = label_mask & semi_hard_mask
if i < 2:
LOG.debug("mask: {}".format(mask))
negative_indexes = compute_semi_hard_indexes(embed[mask], embed_pos[mask], semi_hard_embed)
inputs_neg[np.where(mask)] = semi_hard_input[negative_indexes]
inputs_neg = to_cuda_if_available(inputs_neg)
model_triplet.eval()
with torch.no_grad():
outputs_pos = model_triplet(inputs_pos)
outputs_neg = model_triplet(inputs_neg)
model_triplet.train()
# forward + backward + optimize
outputs = model_triplet(inputs)
return outputs, outputs_pos, outputs_neg
def train(train_loader,
model,
optimizer,
epoch,
weak_mask=None,
strong_mask=None):
class_criterion = nn.BCELoss()
[class_criterion] = to_cuda_if_available([class_criterion])
meters = AverageMeterSet()
meters.update('lr', optimizer.param_groups[0]['lr'])
LOG.debug("Nb batches: {}".format(len(train_loader)))
start = time.time()
for i, (batch_input, target) in enumerate(train_loader):
[batch_input, target] = to_cuda_if_available([batch_input, target])
LOG.debug(batch_input.mean())
strong_pred, weak_pred = model(batch_input)
loss = 0
if weak_mask is not None:
# Weak BCE Loss
# Trick to not take unlabeled data
# Todo figure out another way
target_weak = target.max(-2)[0]
weak_class_loss = class_criterion(weak_pred[weak_mask],
target_weak[weak_mask])
if i == 1:
LOG.debug("target: {}".format(target.mean(-2)))
LOG.debug("Target_weak: {}".format(target_weak))
LOG.debug(weak_class_loss)
meters.update('Weak loss', weak_class_loss.item())
loss += weak_class_loss
if strong_mask is not None:
# Strong BCE loss
strong_class_loss = class_criterion(strong_pred[strong_mask],
target[strong_mask])
meters.update('Strong loss', strong_class_loss.item())
loss += strong_class_loss
assert not (np.isnan(loss.item())
or loss.item() > 1e5), 'Loss explosion: {}'.format(
loss.item())
assert not loss.item() < 0, 'Loss problem, cannot be negative'
meters.update('Loss', loss.item())
# compute gradient and do optimizer step
optimizer.zero_grad()
loss.backward()
optimizer.step()
epoch_time = time.time() - start
LOG.info('Epoch: {}\t'
'Time {:.2f}\t'
'{meters}'.format(epoch, epoch_time, meters=meters))
def calculate_embedding(embedding_dl, model, savedir=None, concatenate=None, squeeze=True):
# If frames, assume the savedir name or the filename is different than when it is not defined
model.eval()
if savedir is not None:
create_folder(savedir)
df = embedding_dl.df.copy()
df.filename = df.filename.apply(lambda x: os.path.join(savedir, os.path.basename(x)))
if savedir is not None:
df.to_csv(os.path.join(savedir, "df"), sep="\t", index=False)
if concatenate is not None:
concat_embed = []
for cnt, (data_in, y) in enumerate(embedding_dl):
data_in = to_cuda_if_available(data_in)
emb = get_embeddings_numpy(data_in, model, flatten=False)
if cnt == 0:
LOG.debug(f"shapes: input: {data_in.shape}, embed: {emb.shape}, dir: {savedir}")
if squeeze:
emb = np.squeeze(emb)
if savedir is not None:
np.save(df.iloc[cnt].filename, emb)
if concatenate == "append":
concat_embed.append(emb)
elif concatenate == "extend":
concat_embed.extend(emb)
else:
if concatenate is not None:
raise NotImplementedError("Impossible to aggregate with this value")
model.train()
if concatenate is not None:
concat_embed = np.array(concat_embed)
return df, concat_embed
return df
def get_predictions(model, valid_dataset, decoder, save_predictions=None):
for i, (input, _) in enumerate(valid_dataset):
[input] = to_cuda_if_available([input])
pred_strong, _ = model(input.unsqueeze(0))
pred_strong = pred_strong.cpu()
pred_strong = pred_strong.squeeze(0).detach().numpy()
if i == 0:
LOG.debug(pred_strong)
pred_strong = ProbabilityEncoder().binarization(pred_strong, binarization_type="global_threshold",
threshold=0.5)
pred_strong = scipy.ndimage.filters.median_filter(pred_strong, (cfg.median_window, 1))
pred = decoder(pred_strong)
pred = pd.DataFrame(pred, columns=["event_label", "onset", "offset"])
pred["filename"] = valid_dataset.filenames.iloc[i]
if i == 0:
LOG.debug("predictions: \n{}".format(pred))
LOG.debug("predictions strong: \n{}".format(pred_strong))
prediction_df = pred.copy()
else:
prediction_df = prediction_df.append(pred)
if save_predictions is not None:
LOG.info("Saving predictions at: {}".format(save_predictions))
prediction_df.to_csv(save_predictions, index=False, sep="\t")
return prediction_df
def get_weak_predictions(model,
valid_dataset,
weak_decoder,
save_predictions=None):
for i, (data, _) in enumerate(valid_dataset):
data = to_cuda_if_available(data)
pred_weak = model(data.unsqueeze(0))
pred_weak = pred_weak.cpu()
pred_weak = pred_weak.squeeze(0).detach().numpy()
if i == 0:
LOG.debug(pred_weak)
pred_weak = ProbabilityEncoder().binarization(
pred_weak, binarization_type="global_threshold", threshold=0.5)
pred = weak_decoder(pred_weak)
pred = pd.DataFrame(pred, columns=["event_labels"])
pred["filename"] = valid_dataset.filenames.iloc[i]
if i == 0:
LOG.debug("predictions: \n{}".format(pred))
prediction_df = pred.copy()
else:
prediction_df = prediction_df.append(pred)
if save_predictions is not None:
LOG.info("Saving predictions at: {}".format(save_predictions))
prediction_df.to_csv(save_predictions, index=False, sep="\t")
return prediction_df
def validate_training_representation(triplet_model, validation_triplets_loader, margin=None, pit=False):
triplet_model.eval()
validation_loss = []
# for counter, triplet_ in enumerate(validation_triplets_loader):
for i, indexes in enumerate(validation_triplets_loader.batch_sampler):
for j, ind in enumerate(indexes):
triplet_ = validation_triplets_loader.dataset[ind]
inputs_, inputs_pos_, inputs_neg_, pred_labels_ = triplet_
inputs_, inputs_pos_, inputs_neg_ = to_cuda_if_available(inputs_, inputs_pos_, inputs_neg_)
out = triplet_model(inputs_)
pos = triplet_model(inputs_pos_)
neg = triplet_model(inputs_neg_)
with torch.no_grad():
dist_pos, dist_neg = get_distances(out, pos, neg,
pit,
)
if margin is not None:
triplet_loss = torch.clamp(margin + dist_pos - dist_neg, min=0.0).mean()
else:
triplet_loss = ratio_loss(dist_pos, dist_neg).mean()
triplet_loss = to_cpu(triplet_loss)
validation_loss.append(triplet_loss.item())
validation_loss = np.mean(validation_loss)
triplet_model.train()
return validation_loss
def train_loop(train_load, model):
loss_bce = []
if args.segment:
for cnt, indexes in enumerate(train_load.batch_sampler):
optimizer.zero_grad()
for j, ind in enumerate(indexes):
inputs, pred_labels = train_set[ind]
if cnt == 0 and epoch_ == 0:
LOG.debug("classif input shape: {}".format(
inputs.shape))
# zero the parameter gradients
inputs, pred_labels = to_cuda_if_available(
inputs, pred_labels)
# forward + backward + optimize
weak_out = model(inputs)
loss_bce = criterion_bce(
weak_out, pred_labels.argmax(0, keepdim=True))
loss_bce.backward()
loss_bce.append(loss_bce.item())
optimizer.step()
else:
for cnt, samples in enumerate(train_load):
optimizer.zero_grad()
inputs, pred_labels = samples
if cnt == 0 and epoch_ == 0:
LOG.debug("classif input shape: {}".format(inputs.shape))
# zero the parameter gradients
inputs, pred_labels = to_cuda_if_available(inputs, pred_labels)
# forward + backward + optimize
weak_out = model(inputs)
loss_bce = criterion_bce(weak_out, pred_labels)
loss_bce.backward()
loss_bce.append(loss_bce.item())
optimizer.step()
loss_bce = np.mean(loss_bce)
print('[%d / %d, %5d] loss: %.3f' %
(epoch_ + 1, n_epochs, cnt + 1, loss_bce))
return loss_bce, model
def test_model(state,
reference_tsv_path,
reduced_number_of_data=None,
strore_predicitions_fname=None):
dataset = DatasetDcase2019Task4(os.path.join(cfg.workspace),
base_feature_dir=os.path.join(
cfg.workspace, "dataset", "features"),
save_log_feature=False)
crnn_kwargs = state["model"]["kwargs"]
crnn = CRNN(**crnn_kwargs)
crnn.load(parameters=state["model"]["state_dict"])
LOG.info("Model loaded at epoch: {}".format(state["epoch"]))
pooling_time_ratio = state["pooling_time_ratio"]
crnn.load(parameters=state["model"]["state_dict"])
scaler = Scaler()
scaler.load_state_dict(state["scaler"])
classes = cfg.classes
many_hot_encoder = ManyHotEncoder.load_state_dict(
state["many_hot_encoder"])
crnn = crnn.eval()
[crnn] = to_cuda_if_available([crnn])
transforms_valid = get_transforms(cfg.max_frames, scaler=scaler)
LOG.info(reference_tsv_path)
df = dataset.initialize_and_get_df(reference_tsv_path,
reduced_number_of_data)
strong_dataload = DataLoadDf(df,
dataset.get_feature_file,
many_hot_encoder.encode_strong_df,
transform=transforms_valid)
predictions = get_predictions(crnn,
strong_dataload,
many_hot_encoder.decode_strong,
pooling_time_ratio,
save_predictions=strore_predicitions_fname)
compute_strong_metrics(predictions, df)
weak_dataload = DataLoadDf(df,
dataset.get_feature_file,
many_hot_encoder.encode_weak,
transform=transforms_valid)
weak_metric = get_f_measure_by_class(
crnn, len(classes), DataLoader(weak_dataload,
batch_size=cfg.batch_size))
LOG.info("Weak F1-score per class: \n {}".format(
pd.DataFrame(weak_metric * 100, many_hot_encoder.labels)))
LOG.info("Weak F1-score macro averaged: {}".format(np.mean(weak_metric)))
drop_last=True, collate_fn=collate_fn)
# #########
# # Model and optimizer
# ########
if resume_training is None:
model_triplet, state = get_model(state, f_args)
optimizer, state = get_optimizer(model_triplet, state)
LOG.info(model_triplet)
pytorch_total_params = sum(p.numel() for p in model_triplet.parameters() if p.requires_grad)
LOG.info("number of parameters in the model: {}".format(pytorch_total_params))
model_triplet.train()
# scheduler = ReduceLROnPlateau(optimizer, 'min', factor=0.1, patience=5, verbose=True)
LOG.info(optimizer)
model_triplet = to_cuda_if_available(model_triplet)
# ##########
# # Callbacks
# ##########
if cfg.save_best:
save_best_call = SaveBest(val_comp="sup")
if cfg.early_stopping is not None:
early_stopping_call = EarlyStopping(patience=cfg.early_stopping, val_comp="sup")
# ##########
# # Training
# ##########
save_results = pd.DataFrame()
model_name_triplet = base_model_name + "triplet"
def train_classifier(train_loader, classif_model, optimizer_classif, many_hot_encoder=None,
valid_loader=None, state={},
dir_model="model", result_path="res", recompute=True):
criterion_bce = nn.BCELoss()
classif_model, criterion_bce = to_cuda_if_available(classif_model, criterion_bce)
print(classif_model)
early_stopping_call = EarlyStopping(patience=cfg.early_stopping, val_comp="sup",
init_patience=cfg.first_early_wait)
save_best_call = SaveBest(val_comp="sup")
# scheduler = ReduceLROnPlateau(optimizer_classif, 'max', factor=0.1, patience=cfg.reduce_lr,
# verbose=True)
print(optimizer_classif)
save_results = pd.DataFrame()
create_folder(dir_model)
if cfg.save_best:
model_path_sup1 = os.path.join(dir_model, "best_model")
else:
model_path_sup1 = os.path.join(dir_model, "epoch_" + str(cfg.n_epoch_classifier))
print("path of model : " + model_path_sup1)
state['many_hot_encoder'] = many_hot_encoder.state_dict()
if not os.path.exists(model_path_sup1) or recompute:
for epoch_ in range(cfg.n_epoch_classifier):
print(classif_model.training)
start = time.time()
loss_mean_bce = []
for i, samples in enumerate(train_loader):
inputs, pred_labels = samples
if i == 0:
LOG.debug("classif input shape: {}".format(inputs.shape))
# zero the parameter gradients
optimizer_classif.zero_grad()
inputs = to_cuda_if_available(inputs)
# forward + backward + optimize
weak_out = classif_model(inputs)
weak_out = to_cpu(weak_out)
# print(output)
loss_bce = criterion_bce(weak_out, pred_labels)
loss_mean_bce.append(loss_bce.item())
loss_bce.backward()
optimizer_classif.step()
loss_mean_bce = np.mean(loss_mean_bce)
classif_model.eval()
n_class = len(many_hot_encoder.labels)
macro_f_measure_train = get_f_measure_by_class(classif_model, n_class,
train_loader)
if valid_loader is not None:
macro_f_measure = get_f_measure_by_class(classif_model, n_class,
valid_loader)
mean_macro_f_measure = np.mean(macro_f_measure)
else:
mean_macro_f_measure = -1
classif_model.train()
print("Time to train an epoch: {}".format(time.time() - start))
# print statistics
print('[%d / %d, %5d] loss: %.3f' %
(epoch_ + 1, cfg.n_epoch_classifier, i + 1, loss_mean_bce))
results = {"train_loss": loss_mean_bce,
"macro_measure_train": np.mean(macro_f_measure_train),
"class_macro_train": np.array_str(macro_f_measure_train, precision=2),
"macro_measure_valid": mean_macro_f_measure,
"class_macro_valid": np.array_str(macro_f_measure, precision=2),
}
for key in results:
LOG.info("\t\t ----> {} : {}".format(key, results[key]))
save_results = save_results.append(results, ignore_index=True)
# scheduler.step(mean_macro_f_measure)
# ##########
# # Callbacks
# ##########
state['epoch'] = epoch_ + 1
state["model"]["state_dict"] = classif_model.state_dict()
state["optimizer"]["state_dict"] = optimizer_classif.state_dict()
state["loss"] = loss_mean_bce
state.update(results)
if cfg.early_stopping is not None:
if early_stopping_call.apply(mean_macro_f_measure):
print("EARLY STOPPING")
break
if cfg.save_best and save_best_call.apply(mean_macro_f_measure):
save_model(state, model_path_sup1)
if cfg.save_best:
LOG.info(
"best model at epoch : {} with macro {}".format(save_best_call.best_epoch, save_best_call.best_val))
LOG.info("loading model from: {}".format(model_path_sup1))
classif_model, state = load_model(model_path_sup1, return_optimizer=False, return_state=True)
else:
model_path_sup1 = os.path.join(dir_model, "epoch_" + str(cfg.n_epoch_classifier))
save_model(state, model_path_sup1)
LOG.debug("model path: {}".format(model_path_sup1))
LOG.debug('Finished Training')
else:
classif_model, state = load_model(model_path_sup1, return_optimizer=False, return_state=True)
LOG.info("#### End classif")
save_results.to_csv(result_path, sep="\t", header=True, index=False)
return classif_model, state
def train(train_loader,
model,
optimizer,
epoch,
ema_model=None,
weak_mask=None,
strong_mask=None):
""" One epoch of a Mean Teacher model
:param train_loader: torch.utils.data.DataLoader, iterator of training batches for an epoch.
Should return 3 values: teacher input, student input, labels
:param model: torch.Module, model to be trained, should return a weak and strong prediction
:param optimizer: torch.Module, optimizer used to train the model
:param epoch: int, the current epoch of training
:param ema_model: torch.Module, student model, should return a weak and strong prediction
:param weak_mask: mask the batch to get only the weak labeled data (used to calculate the loss)
:param strong_mask: mask the batch to get only the strong labeled data (used to calcultate the loss)
"""
class_criterion = nn.BCELoss()
##################################################
class_criterion1 = nn.BCELoss(reduction='none')
##################################################
consistency_criterion = nn.MSELoss()
# [class_criterion, consistency_criterion] = to_cuda_if_available(
# [class_criterion, consistency_criterion])
[class_criterion, class_criterion1,
consistency_criterion] = to_cuda_if_available(
[class_criterion, class_criterion1, consistency_criterion])
meters = AverageMeterSet()
LOG.debug("Nb batches: {}".format(len(train_loader)))
start = time.time()
rampup_length = len(train_loader) * cfg.n_epoch // 2
print("Train\n")
# LOG.info("Weak[k] -> Weak[k]")
# LOG.info("Weak[k] -> strong[k]")
# print(weak_mask.start)
# print(strong_mask.start)
# exit()
count = 0
check_cus_weak = 0
difficulty_loss = 0
loss_w = 1
LOG.info("loss paramater：{}".format(loss_w))
for i, (batch_input, ema_batch_input, target) in enumerate(train_loader):
# print(batch_input.shape)
# print(ema_batch_input.shape)
# exit()
global_step = epoch * len(train_loader) + i
if global_step < rampup_length:
rampup_value = ramps.sigmoid_rampup(global_step, rampup_length)
else:
rampup_value = 1.0
# Todo check if this improves the performance
# adjust_learning_rate(optimizer, rampup_value, rampdown_value)
meters.update('lr', optimizer.param_groups[0]['lr'])
[batch_input, ema_batch_input,
target] = to_cuda_if_available([batch_input, ema_batch_input, target])
LOG.debug("batch_input:{}".format(batch_input.mean()))
# print(batch_input)
# exit()
# Outputs
##################################################
# strong_pred_ema, weak_pred_ema = ema_model(ema_batch_input)
strong_pred_ema, weak_pred_ema, sof_ema = ema_model(ema_batch_input)
sof_ema = sof_ema.detach()
##################################################
strong_pred_ema = strong_pred_ema.detach()
weak_pred_ema = weak_pred_ema.detach()
##################################################
# strong_pred, weak_pred = model(batch_input)
strong_pred, weak_pred, sof = model(batch_input)
##################################################
##################################################
# custom_ema_loss = Custom_BCE_Loss(ema_batch_input, class_criterion1)
if difficulty_loss == 0:
LOG.info("############### Deffine Difficulty Loss ###############")
difficulty_loss = 1
custom_ema_loss = Custom_BCE_Loss_difficulty(ema_batch_input,
class_criterion1,
paramater=loss_w)
custom_ema_loss.initialize(strong_pred_ema, sof_ema)
# custom_loss = Custom_BCE_Loss(batch_input, class_criterion1)
custom_loss = Custom_BCE_Loss_difficulty(batch_input,
class_criterion1,
paramater=loss_w)
custom_loss.initialize(strong_pred, sof)
##################################################
# print(strong_pred.shape)
# print(strong_pred)
# print(weak_pred.shape)
# print(weak_pred)
# exit()
loss = None
# Weak BCE Loss
# Take the max in the time axis
# torch.set_printoptions(threshold=10000)
# print(target[-10])
# # print(target.max(-2))
# # print(target.max(-2)[0])
# print(target.max(-1)[0][-10])
# exit()
target_weak = target.max(-2)[0]
if weak_mask is not None:
weak_class_loss = class_criterion(weak_pred[weak_mask],
target_weak[weak_mask])
ema_class_loss = class_criterion(weak_pred_ema[weak_mask],
target_weak[weak_mask])
print(
"noraml_weak:",
class_criterion(weak_pred[weak_mask], target_weak[weak_mask]))
##################################################
custom_weak_class_loss = custom_loss.weak(target_weak, weak_mask)
custom_ema_class_loss = custom_ema_loss.weak(
target_weak, weak_mask)
print("custom_weak:", custom_weak_class_loss)
##################################################
count += 1
check_cus_weak += custom_weak_class_loss
# print(custom_weak_class_loss.item())
if i == 0:
LOG.debug("target: {}".format(target.mean(-2)))
LOG.debug("Target_weak: {}".format(target_weak))
LOG.debug("Target_weak mask: {}".format(
target_weak[weak_mask]))
LOG.debug(custom_weak_class_loss) ###
LOG.debug("rampup_value: {}".format(rampup_value))
meters.update('weak_class_loss',
custom_weak_class_loss.item()) ###
meters.update('Weak EMA loss', custom_ema_class_loss.item()) ###
# loss = weak_class_loss
loss = custom_weak_class_loss
####################################################################################
# weak_class_loss = class_criterion(strong_pred[weak_mask], target[weak_mask])
# ema_class_loss = class_criterion(strong_pred_ema[weak_mask], target[weak_mask])
# # if i == 0:
# # LOG.debug("target: {}".format(target.mean(-2)))
# # LOG.debug("Target_weak: {}".format(target))
# # LOG.debug("Target_weak mask: {}".format(target[weak_mask]))
# # LOG.debug(weak_class_loss)
# # LOG.debug("rampup_value: {}".format(rampup_value))
# meters.update('weak_class_loss', weak_class_loss.item())
# meters.update('Weak EMA loss', ema_class_loss.item())
# loss = weak_class_loss
####################################################################################
# Strong BCE loss
if strong_mask is not None:
strong_class_loss = class_criterion(strong_pred[strong_mask],
target[strong_mask])
# meters.update('Strong loss', strong_class_loss.item())
strong_ema_class_loss = class_criterion(
strong_pred_ema[strong_mask], target[strong_mask])
# meters.update('Strong EMA loss', strong_ema_class_loss.item())
print(
"normal_strong:",
class_criterion(strong_pred[strong_mask], target[strong_mask]))
##################################################
custom_strong_class_loss = custom_loss.strong(target, strong_mask)
meters.update('Strong loss', custom_strong_class_loss.item())
custom_strong_ema_class_loss = custom_ema_loss.strong(
target, strong_mask)
meters.update('Strong EMA loss',
custom_strong_ema_class_loss.item())
print("custom_strong:", custom_strong_class_loss)
##################################################
if loss is not None:
# loss += strong_class_loss
loss += custom_strong_class_loss
else:
# loss = strong_class_loss
loss = custom_strong_class_loss
# print("check_weak:", class_criterion1(weak_pred[weak_mask], target_weak[weak_mask]).mean())
# print("check_strong:", class_criterion1(strong_pred[strong_mask], target[strong_mask]).mean())
# print("\n")
# exit()
# Teacher-student consistency cost
if ema_model is not None:
consistency_cost = cfg.max_consistency_cost * rampup_value
meters.update('Consistency weight', consistency_cost)
# Take consistency about strong predictions (all data)
consistency_loss_strong = consistency_cost * consistency_criterion(
strong_pred, strong_pred_ema)
meters.update('Consistency strong', consistency_loss_strong.item())
if loss is not None:
loss += consistency_loss_strong
else:
loss = consistency_loss_strong
meters.update('Consistency weight', consistency_cost)
# Take consistency about weak predictions (all data)
consistency_loss_weak = consistency_cost * consistency_criterion(
weak_pred, weak_pred_ema)
meters.update('Consistency weak', consistency_loss_weak.item())
if loss is not None:
loss += consistency_loss_weak
else:
loss = consistency_loss_weak
assert not (np.isnan(loss.item())
or loss.item() > 1e5), 'Loss explosion: {}'.format(
loss.item())
assert not loss.item() < 0, 'Loss problem, cannot be negative'
meters.update('Loss', loss.item())
# compute gradient and do optimizer step
optimizer.zero_grad()
loss.backward()
optimizer.step()
global_step += 1
if ema_model is not None:
update_ema_variables(model, ema_model, 0.999, global_step)
epoch_time = time.time() - start
LOG.info('Epoch: {}\t'
'Time {:.2f}\t'
'{meters}'.format(epoch, epoch_time, meters=meters))
print("\ncheck_cus_weak:\n", check_cus_weak / count)
def test_model(state, reduced_number_of_data, strore_predicitions_fname=None):
crnn_kwargs = state["model"]["kwargs"]
crnn = CRNN(**crnn_kwargs)
crnn.load(parameters=state["model"]["state_dict"])
LOG.info("Model loaded at epoch: {}".format(state["epoch"]))
pooling_time_ratio = state["pooling_time_ratio"]
crnn.load(parameters=state["model"]["state_dict"])
scaler = Scaler()
scaler.load_state_dict(state["scaler"])
classes = cfg.classes
many_hot_encoder = ManyHotEncoder.load_state_dict(
state["many_hot_encoder"])
# ##############
# Validation
# ##############
crnn = crnn.eval()
[crnn] = to_cuda_if_available([crnn])
transforms_valid = get_transforms(cfg.max_frames, scaler=scaler)
# # 2018
# LOG.info("Eval 2018")
# eval_2018_df = dataset.initialize_and_get_df(cfg.eval2018, reduced_number_of_data)
# # Strong
# eval_2018_strong = DataLoadDf(eval_2018_df, dataset.get_feature_file, many_hot_encoder.encode_strong_df,
# transform=transforms_valid)
# predictions = get_predictions(crnn, eval_2018_strong, many_hot_encoder.decode_strong)
# compute_strong_metrics(predictions, eval_2018_df, pooling_time_ratio)
# # Weak
# eval_2018_weak = DataLoadDf(eval_2018_df, dataset.get_feature_file, many_hot_encoder.encode_weak,
# transform=transforms_valid)
# weak_metric = get_f_measure_by_class(crnn, len(classes), DataLoader(eval_2018_weak, batch_size=cfg.batch_size))
# LOG.info("Weak F1-score per class: \n {}".format(pd.DataFrame(weak_metric * 100, many_hot_encoder.labels)))
# LOG.info("Weak F1-score macro averaged: {}".format(np.mean(weak_metric)))
# Validation 2019
# LOG.info("Validation 2019 (original code)")
# b_dataset = B_DatasetDcase2019Task4(cfg.workspace,
# base_feature_dir=os.path.join(cfg.workspace, 'dataset', 'features'),
# save_log_feature=False)
# b_validation_df = b_dataset.initialize_and_get_df(cfg.validation, reduced_number_of_data)
# b_validation_df.to_csv('old.csv')
# b_validation_strong = B_DataLoadDf(b_validation_df,
# b_dataset.get_feature_file, many_hot_encoder.encode_strong_df,
# transform=transforms_valid)
# predictions2 = get_predictions(crnn, b_validation_strong, many_hot_encoder.decode_strong,
# save_predictions=strore_predicitions_fname)
# compute_strong_metrics(predictions2, b_validation_df, pooling_time_ratio)
# b_validation_weak = B_DataLoadDf(b_validation_df, b_dataset.get_feature_file, many_hot_encoder.encode_weak,
# transform=transforms_valid)
# weak_metric = get_f_measure_by_class(crnn, len(classes), DataLoader(b_validation_weak, batch_size=cfg.batch_size))
# LOG.info("Weak F1-score per class: \n {}".format(pd.DataFrame(weak_metric * 100, many_hot_encoder.labels)))
# LOG.info("Weak F1-score macro averaged: {}".format(np.mean(weak_metric)))
# ============================================================================================
# ============================================================================================
# ============================================================================================
dataset = DatasetDcase2019Task4(feature_dir=cfg.feature_dir,
local_path=cfg.workspace,
exp_tag=cfg.exp_tag,
save_log_feature=False)
# Validation 2019
LOG.info("Validation 2019")
validation_df = dataset.initialize_and_get_df(cfg.validation,
reduced_number_of_data)
validation_strong = DataLoadDf(validation_df,
dataset.get_feature_file,
many_hot_encoder.encode_strong_df,
transform=transforms_valid)
predictions = get_predictions(crnn,
validation_strong,
many_hot_encoder.decode_strong,
save_predictions=strore_predicitions_fname)
vdf = validation_df.copy()
vdf.filename = vdf.filename.str.replace('.npy', '.wav')
pdf = predictions.copy()
pdf.filename = pdf.filename.str.replace('.npy', '.wav')
compute_strong_metrics(pdf, vdf, pooling_time_ratio)
validation_weak = DataLoadDf(validation_df,
dataset.get_feature_file,
many_hot_encoder.encode_weak,
transform=transforms_valid)
weak_metric = get_f_measure_by_class(
crnn, len(classes),
DataLoader(validation_weak, batch_size=cfg.batch_size))
LOG.info("Weak F1-score per class: \n {}".format(
pd.DataFrame(weak_metric * 100, many_hot_encoder.labels)))
LOG.info("Weak F1-score macro averaged: {}".format(np.mean(weak_metric)))
},
"pooling_time_ratio": pooling_time_ratio,
"scaler": scaler.state_dict(),
"many_hot_encoder": many_hot_encoder.state_dict()
}
save_best_cb = SaveBest("sup")
# ##############
# Train
# ##############
for epoch in range(cfg.n_epoch):
crnn = crnn.train()
crnn_ema = crnn_ema.train()
[crnn, crnn_ema] = to_cuda_if_available([crnn, crnn_ema])
train(cfg,
training_data,
crnn,
optimizer,
epoch,
ema_model=crnn_ema,
weak_mask=weak_mask,
strong_mask=strong_mask)
crnn = crnn.eval()
LOG.info("\n ### Validation Metrics ### \n")
# predictions = get_predictions(crnn, validation_data, many_hot_encoder.decode_strong,
# save_predictions=None)
# pdf = predictions.copy()
"pooling_time_ratio": pooling_time_ratio,
'scaler': scaler.state_dict(),
"many_hot_encoder": many_hot_encoder.state_dict()
}
save_best_cb = SaveBest("sup")
# Eval 2018
eval_2018_df = dataset.initialize_and_get_df(cfg.eval2018,
reduced_number_of_data)
eval_2018 = DataLoadDf(eval_2018_df,
dataset.get_feature_file,
many_hot_encoder.encode_strong_df,
transform=transforms_valid)
[crnn] = to_cuda_if_available([crnn])
for epoch in range(cfg.n_epoch):
crnn = crnn.train()
train(training_data, crnn, optimizer, epoch, weak_mask, strong_mask)
crnn = crnn.eval()
LOG.info("Training synthetic metric:")
train_predictions = get_predictions(crnn,
train_synth_data,
many_hot_encoder.decode_strong,
pooling_time_ratio,
save_predictions=None)
train_metric = compute_strong_metrics(train_predictions,
train_synth_df)
def get_f_measure_by_class(torch_model,
nb_tags,
dataloader_,
thresholds_=None,
max=False):
""" get f measure for each class given a model and a generator of data (batch_x, y)
Args:
torch_model : Model, model to get predictions, forward should return weak and strong predictions
nb_tags : int, number of classes which are represented
dataloader_ : generator, data generator used to get f_measure
thresholds_ : int or list, thresholds to apply to each class to binarize probabilities
max: bool, whether or not to take the max of the predictions
Returns:
macro_f_measure : list, f measure for each class
"""
torch_model = to_cuda_if_available(torch_model)
# Calculate external metrics
tp = np.zeros(nb_tags)
tn = np.zeros(nb_tags)
fp = np.zeros(nb_tags)
fn = np.zeros(nb_tags)
for counter, (batch_x, y) in enumerate(dataloader_):
if torch.cuda.is_available():
batch_x = batch_x.cuda()
pred_weak = torch_model(batch_x)
pred_weak = pred_weak.cpu().data.numpy()
labels = y.numpy()
# Used only with a model predicting only strong outputs
if len(pred_weak.shape) == 3:
# Max because indicate the presence, give weak labels
pred_weak = np.max(pred_weak, axis=1)
if len(labels.shape) == 3:
labels = np.max(labels, axis=1)
labels = ProbabilityEncoder().binarization(
labels, binarization_type="global_threshold", threshold=0.5)
if counter == 0:
LOG.info(
f"shapes, input: {batch_x.shape}, output: {pred_weak.shape}, label: {labels.shape}"
)
if not max:
if thresholds_ is None:
binarization_type = 'global_threshold'
thresh = 0.5
else:
binarization_type = "class_threshold"
assert type(thresholds_) is list
thresh = thresholds_
batch_predictions = ProbabilityEncoder().binarization(
pred_weak,
binarization_type=binarization_type,
threshold=thresh,
time_axis=0)
else:
batch_predictions = np.zeros(pred_weak.shape)
batch_predictions[:, pred_weak.argmax(1)] = 1
tp_, fp_, fn_, tn_ = intermediate_at_measures(labels,
batch_predictions)
tp += tp_
fp += fp_
fn += fn_
tn += tn_
print("Macro measures: TP: {}\tFP: {}\tFN: {}\tTN: {}".format(
tp, fp, fn, tn))
macro_f_score = np.zeros(nb_tags)
mask_f_score = 2 * tp + fp + fn != 0
macro_f_score[mask_f_score] = 2 * tp[mask_f_score] / (2 * tp + fp +
fn)[mask_f_score]
return macro_f_score
def train(cfg,
train_loader,
model,
optimizer,
epoch,
ema_model=None,
weak_mask=None,
strong_mask=None):
""" One epoch of a Mean Teacher model
:param train_loader: torch.utils.data.DataLoader, iterator of training batches for an epoch.
Should return 3 values: teacher input, student input, labels
:param model: torch.Module, model to be trained, should return a weak and strong prediction
:param optimizer: torch.Module, optimizer used to train the model
:param epoch: int, the current epoch of training
:param ema_model: torch.Module, student model, should return a weak and strong prediction
:param weak_mask: mask the batch to get only the weak labeled data (used to calculate the loss)
:param strong_mask: mask the batch to get only the strong labeled data (used to calcultate the loss)
"""
class_criterion = nn.BCELoss()
consistency_criterion_strong = nn.MSELoss()
lds_criterion = LDSLoss(xi=cfg.vat_xi,
eps=cfg.vat_eps,
n_power_iter=cfg.vat_n_power_iter)
[class_criterion, consistency_criterion_strong,
lds_criterion] = to_cuda_if_available(
[class_criterion, consistency_criterion_strong, lds_criterion])
meters = AverageMeterSet()
LOG.debug("Nb batches: {}".format(len(train_loader)))
start = time.time()
rampup_length = len(train_loader) * cfg.n_epoch // 2
for i, (batch_input, ema_batch_input, target) in enumerate(train_loader):
global_step = epoch * len(train_loader) + i
if global_step < rampup_length:
rampup_value = ramps.sigmoid_rampup(global_step, rampup_length)
else:
rampup_value = 1.0
# Todo check if this improves the performance
# adjust_learning_rate(optimizer, rampup_value, rampdown_value)
meters.update('lr', optimizer.param_groups[0]['lr'])
[batch_input, ema_batch_input,
target] = to_cuda_if_available([batch_input, ema_batch_input, target])
LOG.debug(batch_input.mean())
# Outputs
strong_pred_ema, weak_pred_ema = ema_model(ema_batch_input)
strong_pred_ema = strong_pred_ema.detach()
weak_pred_ema = weak_pred_ema.detach()
strong_pred, weak_pred = model(batch_input)
loss = None
# Weak BCE Loss
# Take the max in axis 2 (assumed to be time)
if len(target.shape) > 2:
target_weak = target.max(-2)[0]
else:
target_weak = target
if weak_mask is not None:
weak_class_loss = class_criterion(weak_pred[weak_mask],
target_weak[weak_mask])
ema_class_loss = class_criterion(weak_pred_ema[weak_mask],
target_weak[weak_mask])
if i == 0:
LOG.debug("target: {}".format(target.mean(-2)))
LOG.debug("Target_weak: {}".format(target_weak))
LOG.debug("Target_weak mask: {}".format(
target_weak[weak_mask]))
LOG.debug(weak_class_loss)
LOG.debug("rampup_value: {}".format(rampup_value))
meters.update('weak_class_loss', weak_class_loss.item())
meters.update('Weak EMA loss', ema_class_loss.item())
loss = weak_class_loss
# Strong BCE loss
if strong_mask is not None:
strong_class_loss = class_criterion(strong_pred[strong_mask],
target[strong_mask])
meters.update('Strong loss', strong_class_loss.item())
strong_ema_class_loss = class_criterion(
strong_pred_ema[strong_mask], target[strong_mask])
meters.update('Strong EMA loss', strong_ema_class_loss.item())
if loss is not None:
loss += strong_class_loss
else:
loss = strong_class_loss
# Teacher-student consistency cost
if ema_model is not None:
consistency_cost = cfg.max_consistency_cost * rampup_value
meters.update('Consistency weight', consistency_cost)
# Take only the consistence with weak and unlabel
consistency_loss_strong = consistency_cost * consistency_criterion_strong(
strong_pred, strong_pred_ema)
meters.update('Consistency strong', consistency_loss_strong.item())
if loss is not None:
loss += consistency_loss_strong
else:
loss = consistency_loss_strong
meters.update('Consistency weight', consistency_cost)
# Take only the consistence with weak and unlabel
consistency_loss_weak = consistency_cost * consistency_criterion_strong(
weak_pred, weak_pred_ema)
meters.update('Consistency weak', consistency_loss_weak.item())
if loss is not None:
loss += consistency_loss_weak
else:
loss = consistency_loss_weak
# LDS loss
if cfg.vat_enabled:
lds_loss = cfg.vat_coeff * lds_criterion(model, batch_input,
weak_pred)
LOG.info('loss: {:.3f}, lds loss: {:.3f}'.format(
loss,
cfg.vat_coeff * lds_loss.detach().cpu().numpy()))
loss += lds_loss
else:
if i % 25 == 0:
LOG.info('loss: {:.3f}'.format(loss))
assert not (np.isnan(loss.item())
or loss.item() > 1e5), 'Loss explosion: {}'.format(
loss.item())
assert not loss.item() < 0, 'Loss problem, cannot be negative'
meters.update('Loss', loss.item())
# compute gradient and do optimizer step
optimizer.zero_grad()
loss.backward()
optimizer.step()
global_step += 1
if ema_model is not None:
update_ema_variables(model, ema_model, 0.999, global_step)
epoch_time = time.time() - start
LOG.info('Epoch: {}\t'
'Time {:.2f}\t'
'{meters}'.format(epoch, epoch_time, meters=meters))
train_set_emb = DataLoadDf(train_weak_df,
many_hot_encoder.encode_weak,
transform=Compose(trans_emb))
valid_set_val = DataLoadDf(valid_weak_df,
many_hot_encoder.encode_weak,
transform=Compose(trans_emb))
test_set_val = DataLoadDf(test_df,
many_hot_encoder.encode_weak,
transform=Compose(trans_emb))
emb_state = {
"scaler": scaler.state_dict(),
"many_hot_encoder": many_hot_encoder.state_dict()
}
emb_model, emb_state = get_model(emb_state, args)
emb_model = to_cuda_if_available(emb_model)
# Classif_model
if args.segment:
X, y = train_set[0]
else:
X, y = next(iter(train_load))
X = to_cuda_if_available(X)
emb = emb_model(X)
LOG.info("shape input CNN: x {}, y {}".format(X.shape, y.shape))
LOG.info("shape after CNN: {}".format(emb.shape))
if args.n_layers_classif == 2:
dimensions = [32, 16]
elif args.n_layers_classif == 1:
dimensions = [32]
classif_args = (emb.shape[-1], dimensions, 10)
"early_stopping": cfg.early_stopping,
"conv_dropout": cfg.conv_dropout,
"frames": cfg.frames_in_sec,
}
params_name.update(args.__dict__)
base_model_name = get_model_name(params_name)
# Model
state = {
"scaler": scaler.state_dict(),
"many_hot_encoder": many_hot_encoder.state_dict(),
"args": vars(args),
}
model, state = get_model(state, args)
optimizer, state = get_optimizer(model, state)
model = to_cuda_if_available(model)
LOG.info(model)
# ##########
# # Callbacks
# ##########
if cfg.save_best:
save_best_call = SaveBest(val_comp="sup")
if cfg.early_stopping is not None:
early_stopping_call = EarlyStopping(patience=cfg.early_stopping,
val_comp="sup")
# lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=20, gamma=0.5)
# x, y = next(iter(train_loader))
x, y = train_set[0]
print("x shape {}, y shape {}".format(x.shape, y.shape))
def datasets_classif(model,
train_weak_embed,
valid_weak_dl_fr,
test_dl_fr,
args,
many_hot_encoder,
classes,
save_name="",
eval_dl=None):
encode_function_label = many_hot_encoder.encode_weak
num_workers = cfg.num_workers
model.eval()
embed_dir = "stored_data/embeddings"
embed_dir = os.path.join(embed_dir, save_name)
create_folder(embed_dir)
fig_dir = os.path.join(embed_dir, "figures")
create_folder(fig_dir)
if args.agg_time is not None:
trans_embedding = [ToTensor(), View(-1)]
else:
trans_embedding = [ToTensor()]
model = to_cuda_if_available(model)
embed_set = "final"
train_embed_dir = os.path.join(embed_dir, embed_set)
df_weak, embed_weak = calculate_embedding(train_weak_embed,
model,
savedir=train_embed_dir,
concatenate="append")
weak_embed = DataLoadDf(df_weak,
encode_function_label,
transform=Compose(trans_embedding))
LOG.info(f"len weak embed: {len(weak_embed)}")
weak_embed.set_transform(Compose(trans_embedding))
batch_size_classif = cfg.batch_size_classif
df_valid, embed_valid = calculate_embedding(valid_weak_dl_fr,
model,
savedir=train_embed_dir,
concatenate="append")
valid_embed = DataLoadDf(df_valid,
encode_function_label,
transform=Compose(trans_embedding))
embed_set = "final_test"
test_embed_dir = os.path.join(embed_dir, embed_set)
df_test_embed, emb_test = calculate_embedding(test_dl_fr,
model,
savedir=test_embed_dir,
concatenate="append")
test_embed = DataLoadDf(df_test_embed,
encode_function_label,
transform=Compose(trans_embedding))
if args.balance:
n_per_class = max(round(batch_size_classif / len(classes)), 1)
weak_sampler = CategoriesSampler(weak_embed.df.event_labels, classes,
n_per_class)
weak_embed_loader = DataLoader(weak_embed,
batch_sampler=weak_sampler,
num_workers=num_workers)
valid_sampler = CategoriesSampler(valid_embed.df.event_labels, classes,
n_per_class)
valid_embed_loader = DataLoader(valid_embed,
batch_sampler=valid_sampler,
num_workers=num_workers)
test_sampler = CategoriesSampler(test_embed.df.event_labels, classes,
n_per_class)
test_embed_loader = DataLoader(test_embed,
batch_sampler=test_sampler,
num_workers=num_workers)
else:
weak_embed_loader = DataLoader(weak_embed,
batch_size=batch_size_classif,
num_workers=num_workers,
shuffle=True,
drop_last=True)
valid_embed_loader = DataLoader(valid_embed,
batch_size=batch_size_classif,
shuffle=False,
num_workers=num_workers,
drop_last=False)
test_embed_loader = DataLoader(test_embed,
batch_size=batch_size_classif,
shuffle=False,
num_workers=num_workers,
drop_last=False)
if eval_dl is not None:
model = to_cuda_if_available(model)
embed_set = "final_eval"
eval_embed_dir = os.path.join(embed_dir, embed_set)
df_eval_embed, embed_eval = calculate_embedding(eval_dl,
model,
savedir=eval_embed_dir,
concatenate="append")
eval_embed = DataLoadDf(df_eval_embed,
encode_function_label,
transform=Compose(trans_embedding))
if args.balance:
eval_sampler = CategoriesSampler(eval_embed.df.event_labels,
classes, n_per_class)
eval_embed_loader = DataLoader(eval_embed,
batch_sampler=eval_sampler,
num_workers=num_workers)
else:
eval_embed_loader = DataLoader(eval_embed,
batch_size=batch_size_classif,
shuffle=False,
num_workers=num_workers,
drop_last=False)
else:
eval_embed_loader = None
model = to_cpu(model)
return {
"train": weak_embed_loader,
"valid": valid_embed_loader,
"test": test_embed_loader,
"eval": eval_embed_loader
}
