def loop_batches(samples, model_triplet, semi_hard_input=None, semi_hard_embed=None, i=0): inputs, inputs_pos, inputs_neg, pred_labels = samples inputs, inputs_pos = to_cuda_if_available(inputs, inputs_pos) if i < 2: LOG.debug("input shape: {}".format(inputs.shape)) if semi_hard_input is not None or semi_hard_embed is not None: assert semi_hard_input is not None, "semi_hard_input and semi_hard_embed should be defined" assert semi_hard_embed is not None, "semi_hard_input and semi_hard_embed should be defined" model_triplet.eval() embed = get_embeddings_numpy(inputs, model_triplet) embed_pos = get_embeddings_numpy(inputs_pos, model_triplet) label_mask = (pred_labels.numpy() == -1).all(-1) semi_hard_mask = np.isnan(inputs_neg.detach().numpy()).reshape(inputs_neg.shape[0], -1).all(-1) mask = label_mask & semi_hard_mask if i < 2: LOG.debug("mask: {}".format(mask)) negative_indexes = compute_semi_hard_indexes(embed[mask], embed_pos[mask], semi_hard_embed) inputs_neg[np.where(mask)] = semi_hard_input[negative_indexes] inputs_neg = to_cuda_if_available(inputs_neg) model_triplet.eval() with torch.no_grad(): outputs_pos = model_triplet(inputs_pos) outputs_neg = model_triplet(inputs_neg) model_triplet.train() # forward + backward + optimize outputs = model_triplet(inputs) return outputs, outputs_pos, outputs_neg
def train(train_loader, model, optimizer, epoch, weak_mask=None, strong_mask=None): class_criterion = nn.BCELoss() [class_criterion] = to_cuda_if_available([class_criterion]) meters = AverageMeterSet() meters.update('lr', optimizer.param_groups[0]['lr']) LOG.debug("Nb batches: {}".format(len(train_loader))) start = time.time() for i, (batch_input, target) in enumerate(train_loader): [batch_input, target] = to_cuda_if_available([batch_input, target]) LOG.debug(batch_input.mean()) strong_pred, weak_pred = model(batch_input) loss = 0 if weak_mask is not None: # Weak BCE Loss # Trick to not take unlabeled data # Todo figure out another way target_weak = target.max(-2)[0] weak_class_loss = class_criterion(weak_pred[weak_mask], target_weak[weak_mask]) if i == 1: LOG.debug("target: {}".format(target.mean(-2))) LOG.debug("Target_weak: {}".format(target_weak)) LOG.debug(weak_class_loss) meters.update('Weak loss', weak_class_loss.item()) loss += weak_class_loss if strong_mask is not None: # Strong BCE loss strong_class_loss = class_criterion(strong_pred[strong_mask], target[strong_mask]) meters.update('Strong loss', strong_class_loss.item()) loss += strong_class_loss assert not (np.isnan(loss.item()) or loss.item() > 1e5), 'Loss explosion: {}'.format( loss.item()) assert not loss.item() < 0, 'Loss problem, cannot be negative' meters.update('Loss', loss.item()) # compute gradient and do optimizer step optimizer.zero_grad() loss.backward() optimizer.step() epoch_time = time.time() - start LOG.info('Epoch: {}\t' 'Time {:.2f}\t' '{meters}'.format(epoch, epoch_time, meters=meters))
def calculate_embedding(embedding_dl, model, savedir=None, concatenate=None, squeeze=True): # If frames, assume the savedir name or the filename is different than when it is not defined model.eval() if savedir is not None: create_folder(savedir) df = embedding_dl.df.copy() df.filename = df.filename.apply(lambda x: os.path.join(savedir, os.path.basename(x))) if savedir is not None: df.to_csv(os.path.join(savedir, "df"), sep="\t", index=False) if concatenate is not None: concat_embed = [] for cnt, (data_in, y) in enumerate(embedding_dl): data_in = to_cuda_if_available(data_in) emb = get_embeddings_numpy(data_in, model, flatten=False) if cnt == 0: LOG.debug(f"shapes: input: {data_in.shape}, embed: {emb.shape}, dir: {savedir}") if squeeze: emb = np.squeeze(emb) if savedir is not None: np.save(df.iloc[cnt].filename, emb) if concatenate == "append": concat_embed.append(emb) elif concatenate == "extend": concat_embed.extend(emb) else: if concatenate is not None: raise NotImplementedError("Impossible to aggregate with this value") model.train() if concatenate is not None: concat_embed = np.array(concat_embed) return df, concat_embed return df
def get_predictions(model, valid_dataset, decoder, save_predictions=None): for i, (input, _) in enumerate(valid_dataset): [input] = to_cuda_if_available([input]) pred_strong, _ = model(input.unsqueeze(0)) pred_strong = pred_strong.cpu() pred_strong = pred_strong.squeeze(0).detach().numpy() if i == 0: LOG.debug(pred_strong) pred_strong = ProbabilityEncoder().binarization(pred_strong, binarization_type="global_threshold", threshold=0.5) pred_strong = scipy.ndimage.filters.median_filter(pred_strong, (cfg.median_window, 1)) pred = decoder(pred_strong) pred = pd.DataFrame(pred, columns=["event_label", "onset", "offset"]) pred["filename"] = valid_dataset.filenames.iloc[i] if i == 0: LOG.debug("predictions: \n{}".format(pred)) LOG.debug("predictions strong: \n{}".format(pred_strong)) prediction_df = pred.copy() else: prediction_df = prediction_df.append(pred) if save_predictions is not None: LOG.info("Saving predictions at: {}".format(save_predictions)) prediction_df.to_csv(save_predictions, index=False, sep="\t") return prediction_df
def get_weak_predictions(model, valid_dataset, weak_decoder, save_predictions=None): for i, (data, _) in enumerate(valid_dataset): data = to_cuda_if_available(data) pred_weak = model(data.unsqueeze(0)) pred_weak = pred_weak.cpu() pred_weak = pred_weak.squeeze(0).detach().numpy() if i == 0: LOG.debug(pred_weak) pred_weak = ProbabilityEncoder().binarization( pred_weak, binarization_type="global_threshold", threshold=0.5) pred = weak_decoder(pred_weak) pred = pd.DataFrame(pred, columns=["event_labels"]) pred["filename"] = valid_dataset.filenames.iloc[i] if i == 0: LOG.debug("predictions: \n{}".format(pred)) prediction_df = pred.copy() else: prediction_df = prediction_df.append(pred) if save_predictions is not None: LOG.info("Saving predictions at: {}".format(save_predictions)) prediction_df.to_csv(save_predictions, index=False, sep="\t") return prediction_df
def validate_training_representation(triplet_model, validation_triplets_loader, margin=None, pit=False): triplet_model.eval() validation_loss = [] # for counter, triplet_ in enumerate(validation_triplets_loader): for i, indexes in enumerate(validation_triplets_loader.batch_sampler): for j, ind in enumerate(indexes): triplet_ = validation_triplets_loader.dataset[ind] inputs_, inputs_pos_, inputs_neg_, pred_labels_ = triplet_ inputs_, inputs_pos_, inputs_neg_ = to_cuda_if_available(inputs_, inputs_pos_, inputs_neg_) out = triplet_model(inputs_) pos = triplet_model(inputs_pos_) neg = triplet_model(inputs_neg_) with torch.no_grad(): dist_pos, dist_neg = get_distances(out, pos, neg, pit, ) if margin is not None: triplet_loss = torch.clamp(margin + dist_pos - dist_neg, min=0.0).mean() else: triplet_loss = ratio_loss(dist_pos, dist_neg).mean() triplet_loss = to_cpu(triplet_loss) validation_loss.append(triplet_loss.item()) validation_loss = np.mean(validation_loss) triplet_model.train() return validation_loss
def train_loop(train_load, model): loss_bce = [] if args.segment: for cnt, indexes in enumerate(train_load.batch_sampler): optimizer.zero_grad() for j, ind in enumerate(indexes): inputs, pred_labels = train_set[ind] if cnt == 0 and epoch_ == 0: LOG.debug("classif input shape: {}".format( inputs.shape)) # zero the parameter gradients inputs, pred_labels = to_cuda_if_available( inputs, pred_labels) # forward + backward + optimize weak_out = model(inputs) loss_bce = criterion_bce( weak_out, pred_labels.argmax(0, keepdim=True)) loss_bce.backward() loss_bce.append(loss_bce.item()) optimizer.step() else: for cnt, samples in enumerate(train_load): optimizer.zero_grad() inputs, pred_labels = samples if cnt == 0 and epoch_ == 0: LOG.debug("classif input shape: {}".format(inputs.shape)) # zero the parameter gradients inputs, pred_labels = to_cuda_if_available(inputs, pred_labels) # forward + backward + optimize weak_out = model(inputs) loss_bce = criterion_bce(weak_out, pred_labels) loss_bce.backward() loss_bce.append(loss_bce.item()) optimizer.step() loss_bce = np.mean(loss_bce) print('[%d / %d, %5d] loss: %.3f' % (epoch_ + 1, n_epochs, cnt + 1, loss_bce)) return loss_bce, model
def test_model(state, reference_tsv_path, reduced_number_of_data=None, strore_predicitions_fname=None): dataset = DatasetDcase2019Task4(os.path.join(cfg.workspace), base_feature_dir=os.path.join( cfg.workspace, "dataset", "features"), save_log_feature=False) crnn_kwargs = state["model"]["kwargs"] crnn = CRNN(**crnn_kwargs) crnn.load(parameters=state["model"]["state_dict"]) LOG.info("Model loaded at epoch: {}".format(state["epoch"])) pooling_time_ratio = state["pooling_time_ratio"] crnn.load(parameters=state["model"]["state_dict"]) scaler = Scaler() scaler.load_state_dict(state["scaler"]) classes = cfg.classes many_hot_encoder = ManyHotEncoder.load_state_dict( state["many_hot_encoder"]) crnn = crnn.eval() [crnn] = to_cuda_if_available([crnn]) transforms_valid = get_transforms(cfg.max_frames, scaler=scaler) LOG.info(reference_tsv_path) df = dataset.initialize_and_get_df(reference_tsv_path, reduced_number_of_data) strong_dataload = DataLoadDf(df, dataset.get_feature_file, many_hot_encoder.encode_strong_df, transform=transforms_valid) predictions = get_predictions(crnn, strong_dataload, many_hot_encoder.decode_strong, pooling_time_ratio, save_predictions=strore_predicitions_fname) compute_strong_metrics(predictions, df) weak_dataload = DataLoadDf(df, dataset.get_feature_file, many_hot_encoder.encode_weak, transform=transforms_valid) weak_metric = get_f_measure_by_class( crnn, len(classes), DataLoader(weak_dataload, batch_size=cfg.batch_size)) LOG.info("Weak F1-score per class: \n {}".format( pd.DataFrame(weak_metric * 100, many_hot_encoder.labels))) LOG.info("Weak F1-score macro averaged: {}".format(np.mean(weak_metric)))
drop_last=True, collate_fn=collate_fn) # ######### # # Model and optimizer # ######## if resume_training is None: model_triplet, state = get_model(state, f_args) optimizer, state = get_optimizer(model_triplet, state) LOG.info(model_triplet) pytorch_total_params = sum(p.numel() for p in model_triplet.parameters() if p.requires_grad) LOG.info("number of parameters in the model: {}".format(pytorch_total_params)) model_triplet.train() # scheduler = ReduceLROnPlateau(optimizer, 'min', factor=0.1, patience=5, verbose=True) LOG.info(optimizer) model_triplet = to_cuda_if_available(model_triplet) # ########## # # Callbacks # ########## if cfg.save_best: save_best_call = SaveBest(val_comp="sup") if cfg.early_stopping is not None: early_stopping_call = EarlyStopping(patience=cfg.early_stopping, val_comp="sup") # ########## # # Training # ########## save_results = pd.DataFrame() model_name_triplet = base_model_name + "triplet"
def train_classifier(train_loader, classif_model, optimizer_classif, many_hot_encoder=None, valid_loader=None, state={}, dir_model="model", result_path="res", recompute=True): criterion_bce = nn.BCELoss() classif_model, criterion_bce = to_cuda_if_available(classif_model, criterion_bce) print(classif_model) early_stopping_call = EarlyStopping(patience=cfg.early_stopping, val_comp="sup", init_patience=cfg.first_early_wait) save_best_call = SaveBest(val_comp="sup") # scheduler = ReduceLROnPlateau(optimizer_classif, 'max', factor=0.1, patience=cfg.reduce_lr, # verbose=True) print(optimizer_classif) save_results = pd.DataFrame() create_folder(dir_model) if cfg.save_best: model_path_sup1 = os.path.join(dir_model, "best_model") else: model_path_sup1 = os.path.join(dir_model, "epoch_" + str(cfg.n_epoch_classifier)) print("path of model : " + model_path_sup1) state['many_hot_encoder'] = many_hot_encoder.state_dict() if not os.path.exists(model_path_sup1) or recompute: for epoch_ in range(cfg.n_epoch_classifier): print(classif_model.training) start = time.time() loss_mean_bce = [] for i, samples in enumerate(train_loader): inputs, pred_labels = samples if i == 0: LOG.debug("classif input shape: {}".format(inputs.shape)) # zero the parameter gradients optimizer_classif.zero_grad() inputs = to_cuda_if_available(inputs) # forward + backward + optimize weak_out = classif_model(inputs) weak_out = to_cpu(weak_out) # print(output) loss_bce = criterion_bce(weak_out, pred_labels) loss_mean_bce.append(loss_bce.item()) loss_bce.backward() optimizer_classif.step() loss_mean_bce = np.mean(loss_mean_bce) classif_model.eval() n_class = len(many_hot_encoder.labels) macro_f_measure_train = get_f_measure_by_class(classif_model, n_class, train_loader) if valid_loader is not None: macro_f_measure = get_f_measure_by_class(classif_model, n_class, valid_loader) mean_macro_f_measure = np.mean(macro_f_measure) else: mean_macro_f_measure = -1 classif_model.train() print("Time to train an epoch: {}".format(time.time() - start)) # print statistics print('[%d / %d, %5d] loss: %.3f' % (epoch_ + 1, cfg.n_epoch_classifier, i + 1, loss_mean_bce)) results = {"train_loss": loss_mean_bce, "macro_measure_train": np.mean(macro_f_measure_train), "class_macro_train": np.array_str(macro_f_measure_train, precision=2), "macro_measure_valid": mean_macro_f_measure, "class_macro_valid": np.array_str(macro_f_measure, precision=2), } for key in results: LOG.info("\t\t ----> {} : {}".format(key, results[key])) save_results = save_results.append(results, ignore_index=True) # scheduler.step(mean_macro_f_measure) # ########## # # Callbacks # ########## state['epoch'] = epoch_ + 1 state["model"]["state_dict"] = classif_model.state_dict() state["optimizer"]["state_dict"] = optimizer_classif.state_dict() state["loss"] = loss_mean_bce state.update(results) if cfg.early_stopping is not None: if early_stopping_call.apply(mean_macro_f_measure): print("EARLY STOPPING") break if cfg.save_best and save_best_call.apply(mean_macro_f_measure): save_model(state, model_path_sup1) if cfg.save_best: LOG.info( "best model at epoch : {} with macro {}".format(save_best_call.best_epoch, save_best_call.best_val)) LOG.info("loading model from: {}".format(model_path_sup1)) classif_model, state = load_model(model_path_sup1, return_optimizer=False, return_state=True) else: model_path_sup1 = os.path.join(dir_model, "epoch_" + str(cfg.n_epoch_classifier)) save_model(state, model_path_sup1) LOG.debug("model path: {}".format(model_path_sup1)) LOG.debug('Finished Training') else: classif_model, state = load_model(model_path_sup1, return_optimizer=False, return_state=True) LOG.info("#### End classif") save_results.to_csv(result_path, sep="\t", header=True, index=False) return classif_model, state
def train(train_loader, model, optimizer, epoch, ema_model=None, weak_mask=None, strong_mask=None): """ One epoch of a Mean Teacher model :param train_loader: torch.utils.data.DataLoader, iterator of training batches for an epoch. Should return 3 values: teacher input, student input, labels :param model: torch.Module, model to be trained, should return a weak and strong prediction :param optimizer: torch.Module, optimizer used to train the model :param epoch: int, the current epoch of training :param ema_model: torch.Module, student model, should return a weak and strong prediction :param weak_mask: mask the batch to get only the weak labeled data (used to calculate the loss) :param strong_mask: mask the batch to get only the strong labeled data (used to calcultate the loss) """ class_criterion = nn.BCELoss() ################################################## class_criterion1 = nn.BCELoss(reduction='none') ################################################## consistency_criterion = nn.MSELoss() # [class_criterion, consistency_criterion] = to_cuda_if_available( # [class_criterion, consistency_criterion]) [class_criterion, class_criterion1, consistency_criterion] = to_cuda_if_available( [class_criterion, class_criterion1, consistency_criterion]) meters = AverageMeterSet() LOG.debug("Nb batches: {}".format(len(train_loader))) start = time.time() rampup_length = len(train_loader) * cfg.n_epoch // 2 print("Train\n") # LOG.info("Weak[k] -> Weak[k]") # LOG.info("Weak[k] -> strong[k]") # print(weak_mask.start) # print(strong_mask.start) # exit() count = 0 check_cus_weak = 0 difficulty_loss = 0 loss_w = 1 LOG.info("loss paramater:{}".format(loss_w)) for i, (batch_input, ema_batch_input, target) in enumerate(train_loader): # print(batch_input.shape) # print(ema_batch_input.shape) # exit() global_step = epoch * len(train_loader) + i if global_step < rampup_length: rampup_value = ramps.sigmoid_rampup(global_step, rampup_length) else: rampup_value = 1.0 # Todo check if this improves the performance # adjust_learning_rate(optimizer, rampup_value, rampdown_value) meters.update('lr', optimizer.param_groups[0]['lr']) [batch_input, ema_batch_input, target] = to_cuda_if_available([batch_input, ema_batch_input, target]) LOG.debug("batch_input:{}".format(batch_input.mean())) # print(batch_input) # exit() # Outputs ################################################## # strong_pred_ema, weak_pred_ema = ema_model(ema_batch_input) strong_pred_ema, weak_pred_ema, sof_ema = ema_model(ema_batch_input) sof_ema = sof_ema.detach() ################################################## strong_pred_ema = strong_pred_ema.detach() weak_pred_ema = weak_pred_ema.detach() ################################################## # strong_pred, weak_pred = model(batch_input) strong_pred, weak_pred, sof = model(batch_input) ################################################## ################################################## # custom_ema_loss = Custom_BCE_Loss(ema_batch_input, class_criterion1) if difficulty_loss == 0: LOG.info("############### Deffine Difficulty Loss ###############") difficulty_loss = 1 custom_ema_loss = Custom_BCE_Loss_difficulty(ema_batch_input, class_criterion1, paramater=loss_w) custom_ema_loss.initialize(strong_pred_ema, sof_ema) # custom_loss = Custom_BCE_Loss(batch_input, class_criterion1) custom_loss = Custom_BCE_Loss_difficulty(batch_input, class_criterion1, paramater=loss_w) custom_loss.initialize(strong_pred, sof) ################################################## # print(strong_pred.shape) # print(strong_pred) # print(weak_pred.shape) # print(weak_pred) # exit() loss = None # Weak BCE Loss # Take the max in the time axis # torch.set_printoptions(threshold=10000) # print(target[-10]) # # print(target.max(-2)) # # print(target.max(-2)[0]) # print(target.max(-1)[0][-10]) # exit() target_weak = target.max(-2)[0] if weak_mask is not None: weak_class_loss = class_criterion(weak_pred[weak_mask], target_weak[weak_mask]) ema_class_loss = class_criterion(weak_pred_ema[weak_mask], target_weak[weak_mask]) print( "noraml_weak:", class_criterion(weak_pred[weak_mask], target_weak[weak_mask])) ################################################## custom_weak_class_loss = custom_loss.weak(target_weak, weak_mask) custom_ema_class_loss = custom_ema_loss.weak( target_weak, weak_mask) print("custom_weak:", custom_weak_class_loss) ################################################## count += 1 check_cus_weak += custom_weak_class_loss # print(custom_weak_class_loss.item()) if i == 0: LOG.debug("target: {}".format(target.mean(-2))) LOG.debug("Target_weak: {}".format(target_weak)) LOG.debug("Target_weak mask: {}".format( target_weak[weak_mask])) LOG.debug(custom_weak_class_loss) ### LOG.debug("rampup_value: {}".format(rampup_value)) meters.update('weak_class_loss', custom_weak_class_loss.item()) ### meters.update('Weak EMA loss', custom_ema_class_loss.item()) ### # loss = weak_class_loss loss = custom_weak_class_loss #################################################################################### # weak_class_loss = class_criterion(strong_pred[weak_mask], target[weak_mask]) # ema_class_loss = class_criterion(strong_pred_ema[weak_mask], target[weak_mask]) # # if i == 0: # # LOG.debug("target: {}".format(target.mean(-2))) # # LOG.debug("Target_weak: {}".format(target)) # # LOG.debug("Target_weak mask: {}".format(target[weak_mask])) # # LOG.debug(weak_class_loss) # # LOG.debug("rampup_value: {}".format(rampup_value)) # meters.update('weak_class_loss', weak_class_loss.item()) # meters.update('Weak EMA loss', ema_class_loss.item()) # loss = weak_class_loss #################################################################################### # Strong BCE loss if strong_mask is not None: strong_class_loss = class_criterion(strong_pred[strong_mask], target[strong_mask]) # meters.update('Strong loss', strong_class_loss.item()) strong_ema_class_loss = class_criterion( strong_pred_ema[strong_mask], target[strong_mask]) # meters.update('Strong EMA loss', strong_ema_class_loss.item()) print( "normal_strong:", class_criterion(strong_pred[strong_mask], target[strong_mask])) ################################################## custom_strong_class_loss = custom_loss.strong(target, strong_mask) meters.update('Strong loss', custom_strong_class_loss.item()) custom_strong_ema_class_loss = custom_ema_loss.strong( target, strong_mask) meters.update('Strong EMA loss', custom_strong_ema_class_loss.item()) print("custom_strong:", custom_strong_class_loss) ################################################## if loss is not None: # loss += strong_class_loss loss += custom_strong_class_loss else: # loss = strong_class_loss loss = custom_strong_class_loss # print("check_weak:", class_criterion1(weak_pred[weak_mask], target_weak[weak_mask]).mean()) # print("check_strong:", class_criterion1(strong_pred[strong_mask], target[strong_mask]).mean()) # print("\n") # exit() # Teacher-student consistency cost if ema_model is not None: consistency_cost = cfg.max_consistency_cost * rampup_value meters.update('Consistency weight', consistency_cost) # Take consistency about strong predictions (all data) consistency_loss_strong = consistency_cost * consistency_criterion( strong_pred, strong_pred_ema) meters.update('Consistency strong', consistency_loss_strong.item()) if loss is not None: loss += consistency_loss_strong else: loss = consistency_loss_strong meters.update('Consistency weight', consistency_cost) # Take consistency about weak predictions (all data) consistency_loss_weak = consistency_cost * consistency_criterion( weak_pred, weak_pred_ema) meters.update('Consistency weak', consistency_loss_weak.item()) if loss is not None: loss += consistency_loss_weak else: loss = consistency_loss_weak assert not (np.isnan(loss.item()) or loss.item() > 1e5), 'Loss explosion: {}'.format( loss.item()) assert not loss.item() < 0, 'Loss problem, cannot be negative' meters.update('Loss', loss.item()) # compute gradient and do optimizer step optimizer.zero_grad() loss.backward() optimizer.step() global_step += 1 if ema_model is not None: update_ema_variables(model, ema_model, 0.999, global_step) epoch_time = time.time() - start LOG.info('Epoch: {}\t' 'Time {:.2f}\t' '{meters}'.format(epoch, epoch_time, meters=meters)) print("\ncheck_cus_weak:\n", check_cus_weak / count)
def test_model(state, reduced_number_of_data, strore_predicitions_fname=None): crnn_kwargs = state["model"]["kwargs"] crnn = CRNN(**crnn_kwargs) crnn.load(parameters=state["model"]["state_dict"]) LOG.info("Model loaded at epoch: {}".format(state["epoch"])) pooling_time_ratio = state["pooling_time_ratio"] crnn.load(parameters=state["model"]["state_dict"]) scaler = Scaler() scaler.load_state_dict(state["scaler"]) classes = cfg.classes many_hot_encoder = ManyHotEncoder.load_state_dict( state["many_hot_encoder"]) # ############## # Validation # ############## crnn = crnn.eval() [crnn] = to_cuda_if_available([crnn]) transforms_valid = get_transforms(cfg.max_frames, scaler=scaler) # # 2018 # LOG.info("Eval 2018") # eval_2018_df = dataset.initialize_and_get_df(cfg.eval2018, reduced_number_of_data) # # Strong # eval_2018_strong = DataLoadDf(eval_2018_df, dataset.get_feature_file, many_hot_encoder.encode_strong_df, # transform=transforms_valid) # predictions = get_predictions(crnn, eval_2018_strong, many_hot_encoder.decode_strong) # compute_strong_metrics(predictions, eval_2018_df, pooling_time_ratio) # # Weak # eval_2018_weak = DataLoadDf(eval_2018_df, dataset.get_feature_file, many_hot_encoder.encode_weak, # transform=transforms_valid) # weak_metric = get_f_measure_by_class(crnn, len(classes), DataLoader(eval_2018_weak, batch_size=cfg.batch_size)) # LOG.info("Weak F1-score per class: \n {}".format(pd.DataFrame(weak_metric * 100, many_hot_encoder.labels))) # LOG.info("Weak F1-score macro averaged: {}".format(np.mean(weak_metric))) # Validation 2019 # LOG.info("Validation 2019 (original code)") # b_dataset = B_DatasetDcase2019Task4(cfg.workspace, # base_feature_dir=os.path.join(cfg.workspace, 'dataset', 'features'), # save_log_feature=False) # b_validation_df = b_dataset.initialize_and_get_df(cfg.validation, reduced_number_of_data) # b_validation_df.to_csv('old.csv') # b_validation_strong = B_DataLoadDf(b_validation_df, # b_dataset.get_feature_file, many_hot_encoder.encode_strong_df, # transform=transforms_valid) # predictions2 = get_predictions(crnn, b_validation_strong, many_hot_encoder.decode_strong, # save_predictions=strore_predicitions_fname) # compute_strong_metrics(predictions2, b_validation_df, pooling_time_ratio) # b_validation_weak = B_DataLoadDf(b_validation_df, b_dataset.get_feature_file, many_hot_encoder.encode_weak, # transform=transforms_valid) # weak_metric = get_f_measure_by_class(crnn, len(classes), DataLoader(b_validation_weak, batch_size=cfg.batch_size)) # LOG.info("Weak F1-score per class: \n {}".format(pd.DataFrame(weak_metric * 100, many_hot_encoder.labels))) # LOG.info("Weak F1-score macro averaged: {}".format(np.mean(weak_metric))) # ============================================================================================ # ============================================================================================ # ============================================================================================ dataset = DatasetDcase2019Task4(feature_dir=cfg.feature_dir, local_path=cfg.workspace, exp_tag=cfg.exp_tag, save_log_feature=False) # Validation 2019 LOG.info("Validation 2019") validation_df = dataset.initialize_and_get_df(cfg.validation, reduced_number_of_data) validation_strong = DataLoadDf(validation_df, dataset.get_feature_file, many_hot_encoder.encode_strong_df, transform=transforms_valid) predictions = get_predictions(crnn, validation_strong, many_hot_encoder.decode_strong, save_predictions=strore_predicitions_fname) vdf = validation_df.copy() vdf.filename = vdf.filename.str.replace('.npy', '.wav') pdf = predictions.copy() pdf.filename = pdf.filename.str.replace('.npy', '.wav') compute_strong_metrics(pdf, vdf, pooling_time_ratio) validation_weak = DataLoadDf(validation_df, dataset.get_feature_file, many_hot_encoder.encode_weak, transform=transforms_valid) weak_metric = get_f_measure_by_class( crnn, len(classes), DataLoader(validation_weak, batch_size=cfg.batch_size)) LOG.info("Weak F1-score per class: \n {}".format( pd.DataFrame(weak_metric * 100, many_hot_encoder.labels))) LOG.info("Weak F1-score macro averaged: {}".format(np.mean(weak_metric)))
}, "pooling_time_ratio": pooling_time_ratio, "scaler": scaler.state_dict(), "many_hot_encoder": many_hot_encoder.state_dict() } save_best_cb = SaveBest("sup") # ############## # Train # ############## for epoch in range(cfg.n_epoch): crnn = crnn.train() crnn_ema = crnn_ema.train() [crnn, crnn_ema] = to_cuda_if_available([crnn, crnn_ema]) train(cfg, training_data, crnn, optimizer, epoch, ema_model=crnn_ema, weak_mask=weak_mask, strong_mask=strong_mask) crnn = crnn.eval() LOG.info("\n ### Validation Metrics ### \n") # predictions = get_predictions(crnn, validation_data, many_hot_encoder.decode_strong, # save_predictions=None) # pdf = predictions.copy()
"pooling_time_ratio": pooling_time_ratio, 'scaler': scaler.state_dict(), "many_hot_encoder": many_hot_encoder.state_dict() } save_best_cb = SaveBest("sup") # Eval 2018 eval_2018_df = dataset.initialize_and_get_df(cfg.eval2018, reduced_number_of_data) eval_2018 = DataLoadDf(eval_2018_df, dataset.get_feature_file, many_hot_encoder.encode_strong_df, transform=transforms_valid) [crnn] = to_cuda_if_available([crnn]) for epoch in range(cfg.n_epoch): crnn = crnn.train() train(training_data, crnn, optimizer, epoch, weak_mask, strong_mask) crnn = crnn.eval() LOG.info("Training synthetic metric:") train_predictions = get_predictions(crnn, train_synth_data, many_hot_encoder.decode_strong, pooling_time_ratio, save_predictions=None) train_metric = compute_strong_metrics(train_predictions, train_synth_df)
def get_f_measure_by_class(torch_model, nb_tags, dataloader_, thresholds_=None, max=False): """ get f measure for each class given a model and a generator of data (batch_x, y) Args: torch_model : Model, model to get predictions, forward should return weak and strong predictions nb_tags : int, number of classes which are represented dataloader_ : generator, data generator used to get f_measure thresholds_ : int or list, thresholds to apply to each class to binarize probabilities max: bool, whether or not to take the max of the predictions Returns: macro_f_measure : list, f measure for each class """ torch_model = to_cuda_if_available(torch_model) # Calculate external metrics tp = np.zeros(nb_tags) tn = np.zeros(nb_tags) fp = np.zeros(nb_tags) fn = np.zeros(nb_tags) for counter, (batch_x, y) in enumerate(dataloader_): if torch.cuda.is_available(): batch_x = batch_x.cuda() pred_weak = torch_model(batch_x) pred_weak = pred_weak.cpu().data.numpy() labels = y.numpy() # Used only with a model predicting only strong outputs if len(pred_weak.shape) == 3: # Max because indicate the presence, give weak labels pred_weak = np.max(pred_weak, axis=1) if len(labels.shape) == 3: labels = np.max(labels, axis=1) labels = ProbabilityEncoder().binarization( labels, binarization_type="global_threshold", threshold=0.5) if counter == 0: LOG.info( f"shapes, input: {batch_x.shape}, output: {pred_weak.shape}, label: {labels.shape}" ) if not max: if thresholds_ is None: binarization_type = 'global_threshold' thresh = 0.5 else: binarization_type = "class_threshold" assert type(thresholds_) is list thresh = thresholds_ batch_predictions = ProbabilityEncoder().binarization( pred_weak, binarization_type=binarization_type, threshold=thresh, time_axis=0) else: batch_predictions = np.zeros(pred_weak.shape) batch_predictions[:, pred_weak.argmax(1)] = 1 tp_, fp_, fn_, tn_ = intermediate_at_measures(labels, batch_predictions) tp += tp_ fp += fp_ fn += fn_ tn += tn_ print("Macro measures: TP: {}\tFP: {}\tFN: {}\tTN: {}".format( tp, fp, fn, tn)) macro_f_score = np.zeros(nb_tags) mask_f_score = 2 * tp + fp + fn != 0 macro_f_score[mask_f_score] = 2 * tp[mask_f_score] / (2 * tp + fp + fn)[mask_f_score] return macro_f_score
def train(cfg, train_loader, model, optimizer, epoch, ema_model=None, weak_mask=None, strong_mask=None): """ One epoch of a Mean Teacher model :param train_loader: torch.utils.data.DataLoader, iterator of training batches for an epoch. Should return 3 values: teacher input, student input, labels :param model: torch.Module, model to be trained, should return a weak and strong prediction :param optimizer: torch.Module, optimizer used to train the model :param epoch: int, the current epoch of training :param ema_model: torch.Module, student model, should return a weak and strong prediction :param weak_mask: mask the batch to get only the weak labeled data (used to calculate the loss) :param strong_mask: mask the batch to get only the strong labeled data (used to calcultate the loss) """ class_criterion = nn.BCELoss() consistency_criterion_strong = nn.MSELoss() lds_criterion = LDSLoss(xi=cfg.vat_xi, eps=cfg.vat_eps, n_power_iter=cfg.vat_n_power_iter) [class_criterion, consistency_criterion_strong, lds_criterion] = to_cuda_if_available( [class_criterion, consistency_criterion_strong, lds_criterion]) meters = AverageMeterSet() LOG.debug("Nb batches: {}".format(len(train_loader))) start = time.time() rampup_length = len(train_loader) * cfg.n_epoch // 2 for i, (batch_input, ema_batch_input, target) in enumerate(train_loader): global_step = epoch * len(train_loader) + i if global_step < rampup_length: rampup_value = ramps.sigmoid_rampup(global_step, rampup_length) else: rampup_value = 1.0 # Todo check if this improves the performance # adjust_learning_rate(optimizer, rampup_value, rampdown_value) meters.update('lr', optimizer.param_groups[0]['lr']) [batch_input, ema_batch_input, target] = to_cuda_if_available([batch_input, ema_batch_input, target]) LOG.debug(batch_input.mean()) # Outputs strong_pred_ema, weak_pred_ema = ema_model(ema_batch_input) strong_pred_ema = strong_pred_ema.detach() weak_pred_ema = weak_pred_ema.detach() strong_pred, weak_pred = model(batch_input) loss = None # Weak BCE Loss # Take the max in axis 2 (assumed to be time) if len(target.shape) > 2: target_weak = target.max(-2)[0] else: target_weak = target if weak_mask is not None: weak_class_loss = class_criterion(weak_pred[weak_mask], target_weak[weak_mask]) ema_class_loss = class_criterion(weak_pred_ema[weak_mask], target_weak[weak_mask]) if i == 0: LOG.debug("target: {}".format(target.mean(-2))) LOG.debug("Target_weak: {}".format(target_weak)) LOG.debug("Target_weak mask: {}".format( target_weak[weak_mask])) LOG.debug(weak_class_loss) LOG.debug("rampup_value: {}".format(rampup_value)) meters.update('weak_class_loss', weak_class_loss.item()) meters.update('Weak EMA loss', ema_class_loss.item()) loss = weak_class_loss # Strong BCE loss if strong_mask is not None: strong_class_loss = class_criterion(strong_pred[strong_mask], target[strong_mask]) meters.update('Strong loss', strong_class_loss.item()) strong_ema_class_loss = class_criterion( strong_pred_ema[strong_mask], target[strong_mask]) meters.update('Strong EMA loss', strong_ema_class_loss.item()) if loss is not None: loss += strong_class_loss else: loss = strong_class_loss # Teacher-student consistency cost if ema_model is not None: consistency_cost = cfg.max_consistency_cost * rampup_value meters.update('Consistency weight', consistency_cost) # Take only the consistence with weak and unlabel consistency_loss_strong = consistency_cost * consistency_criterion_strong( strong_pred, strong_pred_ema) meters.update('Consistency strong', consistency_loss_strong.item()) if loss is not None: loss += consistency_loss_strong else: loss = consistency_loss_strong meters.update('Consistency weight', consistency_cost) # Take only the consistence with weak and unlabel consistency_loss_weak = consistency_cost * consistency_criterion_strong( weak_pred, weak_pred_ema) meters.update('Consistency weak', consistency_loss_weak.item()) if loss is not None: loss += consistency_loss_weak else: loss = consistency_loss_weak # LDS loss if cfg.vat_enabled: lds_loss = cfg.vat_coeff * lds_criterion(model, batch_input, weak_pred) LOG.info('loss: {:.3f}, lds loss: {:.3f}'.format( loss, cfg.vat_coeff * lds_loss.detach().cpu().numpy())) loss += lds_loss else: if i % 25 == 0: LOG.info('loss: {:.3f}'.format(loss)) assert not (np.isnan(loss.item()) or loss.item() > 1e5), 'Loss explosion: {}'.format( loss.item()) assert not loss.item() < 0, 'Loss problem, cannot be negative' meters.update('Loss', loss.item()) # compute gradient and do optimizer step optimizer.zero_grad() loss.backward() optimizer.step() global_step += 1 if ema_model is not None: update_ema_variables(model, ema_model, 0.999, global_step) epoch_time = time.time() - start LOG.info('Epoch: {}\t' 'Time {:.2f}\t' '{meters}'.format(epoch, epoch_time, meters=meters))
train_set_emb = DataLoadDf(train_weak_df, many_hot_encoder.encode_weak, transform=Compose(trans_emb)) valid_set_val = DataLoadDf(valid_weak_df, many_hot_encoder.encode_weak, transform=Compose(trans_emb)) test_set_val = DataLoadDf(test_df, many_hot_encoder.encode_weak, transform=Compose(trans_emb)) emb_state = { "scaler": scaler.state_dict(), "many_hot_encoder": many_hot_encoder.state_dict() } emb_model, emb_state = get_model(emb_state, args) emb_model = to_cuda_if_available(emb_model) # Classif_model if args.segment: X, y = train_set[0] else: X, y = next(iter(train_load)) X = to_cuda_if_available(X) emb = emb_model(X) LOG.info("shape input CNN: x {}, y {}".format(X.shape, y.shape)) LOG.info("shape after CNN: {}".format(emb.shape)) if args.n_layers_classif == 2: dimensions = [32, 16] elif args.n_layers_classif == 1: dimensions = [32] classif_args = (emb.shape[-1], dimensions, 10)
"early_stopping": cfg.early_stopping, "conv_dropout": cfg.conv_dropout, "frames": cfg.frames_in_sec, } params_name.update(args.__dict__) base_model_name = get_model_name(params_name) # Model state = { "scaler": scaler.state_dict(), "many_hot_encoder": many_hot_encoder.state_dict(), "args": vars(args), } model, state = get_model(state, args) optimizer, state = get_optimizer(model, state) model = to_cuda_if_available(model) LOG.info(model) # ########## # # Callbacks # ########## if cfg.save_best: save_best_call = SaveBest(val_comp="sup") if cfg.early_stopping is not None: early_stopping_call = EarlyStopping(patience=cfg.early_stopping, val_comp="sup") # lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=20, gamma=0.5) # x, y = next(iter(train_loader)) x, y = train_set[0] print("x shape {}, y shape {}".format(x.shape, y.shape))
def datasets_classif(model, train_weak_embed, valid_weak_dl_fr, test_dl_fr, args, many_hot_encoder, classes, save_name="", eval_dl=None): encode_function_label = many_hot_encoder.encode_weak num_workers = cfg.num_workers model.eval() embed_dir = "stored_data/embeddings" embed_dir = os.path.join(embed_dir, save_name) create_folder(embed_dir) fig_dir = os.path.join(embed_dir, "figures") create_folder(fig_dir) if args.agg_time is not None: trans_embedding = [ToTensor(), View(-1)] else: trans_embedding = [ToTensor()] model = to_cuda_if_available(model) embed_set = "final" train_embed_dir = os.path.join(embed_dir, embed_set) df_weak, embed_weak = calculate_embedding(train_weak_embed, model, savedir=train_embed_dir, concatenate="append") weak_embed = DataLoadDf(df_weak, encode_function_label, transform=Compose(trans_embedding)) LOG.info(f"len weak embed: {len(weak_embed)}") weak_embed.set_transform(Compose(trans_embedding)) batch_size_classif = cfg.batch_size_classif df_valid, embed_valid = calculate_embedding(valid_weak_dl_fr, model, savedir=train_embed_dir, concatenate="append") valid_embed = DataLoadDf(df_valid, encode_function_label, transform=Compose(trans_embedding)) embed_set = "final_test" test_embed_dir = os.path.join(embed_dir, embed_set) df_test_embed, emb_test = calculate_embedding(test_dl_fr, model, savedir=test_embed_dir, concatenate="append") test_embed = DataLoadDf(df_test_embed, encode_function_label, transform=Compose(trans_embedding)) if args.balance: n_per_class = max(round(batch_size_classif / len(classes)), 1) weak_sampler = CategoriesSampler(weak_embed.df.event_labels, classes, n_per_class) weak_embed_loader = DataLoader(weak_embed, batch_sampler=weak_sampler, num_workers=num_workers) valid_sampler = CategoriesSampler(valid_embed.df.event_labels, classes, n_per_class) valid_embed_loader = DataLoader(valid_embed, batch_sampler=valid_sampler, num_workers=num_workers) test_sampler = CategoriesSampler(test_embed.df.event_labels, classes, n_per_class) test_embed_loader = DataLoader(test_embed, batch_sampler=test_sampler, num_workers=num_workers) else: weak_embed_loader = DataLoader(weak_embed, batch_size=batch_size_classif, num_workers=num_workers, shuffle=True, drop_last=True) valid_embed_loader = DataLoader(valid_embed, batch_size=batch_size_classif, shuffle=False, num_workers=num_workers, drop_last=False) test_embed_loader = DataLoader(test_embed, batch_size=batch_size_classif, shuffle=False, num_workers=num_workers, drop_last=False) if eval_dl is not None: model = to_cuda_if_available(model) embed_set = "final_eval" eval_embed_dir = os.path.join(embed_dir, embed_set) df_eval_embed, embed_eval = calculate_embedding(eval_dl, model, savedir=eval_embed_dir, concatenate="append") eval_embed = DataLoadDf(df_eval_embed, encode_function_label, transform=Compose(trans_embedding)) if args.balance: eval_sampler = CategoriesSampler(eval_embed.df.event_labels, classes, n_per_class) eval_embed_loader = DataLoader(eval_embed, batch_sampler=eval_sampler, num_workers=num_workers) else: eval_embed_loader = DataLoader(eval_embed, batch_size=batch_size_classif, shuffle=False, num_workers=num_workers, drop_last=False) else: eval_embed_loader = None model = to_cpu(model) return { "train": weak_embed_loader, "valid": valid_embed_loader, "test": test_embed_loader, "eval": eval_embed_loader }