def valStep(dataLoader, cpcModel, cpcCriterion): cpcCriterion.eval() cpcModel.eval() logs = {} cpcCriterion.eval() cpcModel.eval() iter = 0 for step, fulldata in enumerate(dataLoader): batchData, labelData = fulldata label = labelData['speaker'] batchData = batchData.cuda(non_blocking=True) label = label.cuda(non_blocking=True) with torch.no_grad(): c_feature, encoded_data, label = cpcModel(batchData, label) allLosses, allAcc, _ = cpcCriterion(c_feature, encoded_data, label, None) if "locLoss_val" not in logs: logs["locLoss_val"] = np.zeros(allLosses.size(1)) logs["locAcc_val"] = np.zeros(allLosses.size(1)) iter += 1 logs["locLoss_val"] += allLosses.mean(dim=0).cpu().numpy() logs["locAcc_val"] += allAcc.mean(dim=0).cpu().numpy() logs = utils.update_logs(logs, iter) logs["iter"] = iter utils.show_logs("Validation loss:", logs) return logs
def trainStep(dataLoader, cpcModel, cpcCriterion, optimizer, scheduler, loggingStep): cpcModel.train() cpcCriterion.train() start_time = time.perf_counter() n_examples = 0 logs, lastlogs = {}, None iter = 0 for step, fulldata in enumerate(dataLoader): batchData, labelData = fulldata label = labelData['speaker'] n_examples += batchData.size(0) batchData = batchData.cuda(non_blocking=True) label = label.cuda(non_blocking=True) c_feature, encoded_data, label = cpcModel(batchData, label) allLosses, allAcc, _ = cpcCriterion(c_feature, encoded_data, label, None) totLoss = allLosses.sum() totLoss.backward() # Show grads ? optimizer.step() optimizer.zero_grad() if "locLoss_train" not in logs: logs["locLoss_train"] = np.zeros(allLosses.size(1)) logs["locAcc_train"] = np.zeros(allLosses.size(1)) iter += 1 logs["locLoss_train"] += (allLosses.mean(dim=0)).detach().cpu().numpy() logs["locAcc_train"] += (allAcc.mean(dim=0)).cpu().numpy() if (step + 1) % loggingStep == 0: new_time = time.perf_counter() elapsed = new_time - start_time print(f"Update {step + 1}") print(f"elapsed: {elapsed:.1f} s") print( f"{1000.0 * elapsed / loggingStep:.1f} ms per batch, {1000.0 * elapsed / n_examples:.1f} ms / example") locLogs = utils.update_logs(logs, loggingStep, lastlogs) lastlogs = deepcopy(logs) utils.show_logs("Training loss", locLogs) start_time, n_examples = new_time, 0 if scheduler is not None: scheduler.step() logs = utils.update_logs(logs, iter) logs["iter"] = iter utils.show_logs("Average training loss on epoch", logs) return logs
def run(feature_maker, criterion, train_loader, val_loader, optimizer, logs, n_epochs, path_checkpoint, label_key="speaker", centerpushSettings=None): start_epoch = len(logs["epoch"]) best_acc = -1 start_time = time.time() for epoch in range(start_epoch, n_epochs): logs_train = train_step(feature_maker, criterion, train_loader, optimizer, label_key=label_key, centerpushSettings=centerpushSettings) logs_val = val_step(feature_maker, criterion, val_loader, label_key=label_key, centerpushSettings=centerpushSettings) print('') print('_'*50) print(f'Ran {epoch + 1} epochs ' f'in {time.time() - start_time:.2f} seconds') utils.show_logs("Training loss", logs_train) utils.show_logs("Validation loss", logs_val) print('_'*50) print('') if logs_val["locAcc_val"] > best_acc: best_state = deepcopy(fl.get_module(feature_maker).state_dict()) best_acc = logs_val["locAcc_val"] logs["epoch"].append(epoch) for key, value in dict(logs_train, **logs_val).items(): if key not in logs: logs[key] = [None for x in range(epoch)] if isinstance(value, np.ndarray): value = value.tolist() logs[key].append(value) if (epoch % logs["saveStep"] == 0 and epoch > 0) or epoch == n_epochs - 1: model_state_dict = fl.get_module(feature_maker).state_dict() criterion_state_dict = fl.get_module(criterion).state_dict() fl.save_checkpoint(model_state_dict, criterion_state_dict, optimizer.state_dict(), best_state, f"{path_checkpoint}_{epoch}.pt") utils.save_logs(logs, f"{path_checkpoint}_logs.json")
logs = {"epoch": []} optimizer = torch.optim.Adam(list(clf.parameters()), lr=2e-3, betas=(0.9, 0.999), eps=2e-8) best_acc = -1 start_time = time.time() for ep in range(MAX_EPOCHS): logs_train = train_step(feat_gen, clf, train_loader, optimizer, ep) #logs_val = val_step(feature_maker, criterion, val_loader) #logs = {"epoch": [], "iter": [], "saveStep": -1} # saveStep=-1, save only best checkpoint! logs_test = test_step(feat_gen, clf, test_loader, optimizer, ep) print('') print('_' * 50) print(f'Ran {ep + 1} epochs ' f'in {time.time() - start_time:.2f} seconds') utils.show_logs("Training loss", logs_train) utils.show_logs("Test loss", logs_test) print('_' * 50) print('') if logs_test["locAcc_test"] > best_acc: best_state = deepcopy(fl.get_module(feat_gen).state_dict()) best_acc = logs_test["locAcc_test"] logs["epoch"].append(ep) for key, value in dict(logs_train, **logs_test).items(): if key not in logs: logs[key] = [None for x in range(ep)] if isinstance(value, np.ndarray): value = value.tolist() logs[key].append(value)
def trainLinsepClassification( feature_maker, criterion, # combined with classification model before train_loader, val_loader, optimizer, path_logs, logs_save_step, path_best_checkpoint, n_epochs, cpc_epoch, label_key="speaker", centerpushSettings=None): wasOptimizeCPC = feature_maker.optimize if hasattr(feature_maker, 'optimize') else None feature_maker.eval() feature_maker.optimize = False start_epoch = 0 best_train_acc = -1 best_acc = -1 bect_epoch = -1 logs = {"epoch": [], "iter": [], "saveStep": logs_save_step} start_time = time.time() for epoch in range(start_epoch, n_epochs): logs_train = train_step(feature_maker, criterion, train_loader, optimizer, label_key, centerpushSettings=centerpushSettings) logs_val = val_step(feature_maker, criterion, val_loader, label_key, centerpushSettings=centerpushSettings) print('') print('_'*50) print(f'Ran {epoch + 1} {label_key} classification epochs ' f'in {time.time() - start_time:.2f} seconds') utils.show_logs("Training loss", logs_train) utils.show_logs("Validation loss", logs_val) print('_'*50) print('') if logs_val["locAcc_val"] > best_acc: best_state_cpc = deepcopy(fl.get_module(feature_maker).state_dict()) best_state_classif_crit = deepcopy(fl.get_module(criterion).state_dict()) optimizer_state_best_ep = optimizer.state_dict() best_epoch = epoch best_acc = logs_val["locAcc_val"] if logs_train["locAcc_train"] > best_train_acc: best_train_acc = logs_train["locAcc_train"] logs["epoch"].append(epoch) for key, value in dict(logs_train, **logs_val).items(): if key not in logs: logs[key] = [None for x in range(epoch)] if isinstance(value, np.ndarray): value = value.tolist() logs[key].append(value) if (epoch % logs["saveStep"] == 0 and epoch > 0) or epoch == n_epochs - 1: model_state_dict = fl.get_module(feature_maker).state_dict() criterion_state_dict = fl.get_module(criterion).state_dict() # fl.save_checkpoint(model_state_dict, criterion_state_dict, # optimizer.state_dict(), best_state, # f"{path_checkpoint}_{epoch}.pt") utils.save_logs(logs, f"{path_logs}_logs.json") if path_best_checkpoint: save_linsep_best_checkpoint(best_state_cpc, best_state_classif_crit, optimizer_state_best_ep, # TODO check if should save that epoch or last in optimizer os.path.join(path_best_checkpoint, f"{label_key}_classif_best-epoch{best_epoch}-cpc_epoch{cpc_epoch}.pt")) feature_maker.optimize = wasOptimizeCPC return {'num_epoch_trained': n_epochs, 'best_val_acc': best_acc, 'best_train_acc': best_train_acc }