def valStep(dataLoader,
cpcModel,
cpcCriterion):
cpcCriterion.eval()
cpcModel.eval()
logs = {}
cpcCriterion.eval()
cpcModel.eval()
iter = 0
for step, fulldata in enumerate(dataLoader):
batchData, labelData = fulldata
label = labelData['speaker']
batchData = batchData.cuda(non_blocking=True)
label = label.cuda(non_blocking=True)
with torch.no_grad():
c_feature, encoded_data, label = cpcModel(batchData, label)
allLosses, allAcc, _ = cpcCriterion(c_feature, encoded_data, label, None)
if "locLoss_val" not in logs:
logs["locLoss_val"] = np.zeros(allLosses.size(1))
logs["locAcc_val"] = np.zeros(allLosses.size(1))
iter += 1
logs["locLoss_val"] += allLosses.mean(dim=0).cpu().numpy()
logs["locAcc_val"] += allAcc.mean(dim=0).cpu().numpy()
logs = utils.update_logs(logs, iter)
logs["iter"] = iter
utils.show_logs("Validation loss:", logs)
return logs
def trainStep(dataLoader,
cpcModel,
cpcCriterion,
optimizer,
scheduler,
loggingStep):
cpcModel.train()
cpcCriterion.train()
start_time = time.perf_counter()
n_examples = 0
logs, lastlogs = {}, None
iter = 0
for step, fulldata in enumerate(dataLoader):
batchData, labelData = fulldata
label = labelData['speaker']
n_examples += batchData.size(0)
batchData = batchData.cuda(non_blocking=True)
label = label.cuda(non_blocking=True)
c_feature, encoded_data, label = cpcModel(batchData, label)
allLosses, allAcc, _ = cpcCriterion(c_feature, encoded_data, label, None)
totLoss = allLosses.sum()
totLoss.backward()
# Show grads ?
optimizer.step()
optimizer.zero_grad()
if "locLoss_train" not in logs:
logs["locLoss_train"] = np.zeros(allLosses.size(1))
logs["locAcc_train"] = np.zeros(allLosses.size(1))
iter += 1
logs["locLoss_train"] += (allLosses.mean(dim=0)).detach().cpu().numpy()
logs["locAcc_train"] += (allAcc.mean(dim=0)).cpu().numpy()
if (step + 1) % loggingStep == 0:
new_time = time.perf_counter()
elapsed = new_time - start_time
print(f"Update {step + 1}")
print(f"elapsed: {elapsed:.1f} s")
print(
f"{1000.0 * elapsed / loggingStep:.1f} ms per batch, {1000.0 * elapsed / n_examples:.1f} ms / example")
locLogs = utils.update_logs(logs, loggingStep, lastlogs)
lastlogs = deepcopy(logs)
utils.show_logs("Training loss", locLogs)
start_time, n_examples = new_time, 0
if scheduler is not None:
scheduler.step()
logs = utils.update_logs(logs, iter)
logs["iter"] = iter
utils.show_logs("Average training loss on epoch", logs)
return logs
def run(feature_maker,
criterion,
train_loader,
val_loader,
optimizer,
logs,
n_epochs,
path_checkpoint,
label_key="speaker",
centerpushSettings=None):
start_epoch = len(logs["epoch"])
best_acc = -1
start_time = time.time()
for epoch in range(start_epoch, n_epochs):
logs_train = train_step(feature_maker, criterion, train_loader,
optimizer, label_key=label_key, centerpushSettings=centerpushSettings)
logs_val = val_step(feature_maker, criterion, val_loader, label_key=label_key, centerpushSettings=centerpushSettings)
print('')
print('_'*50)
print(f'Ran {epoch + 1} epochs '
f'in {time.time() - start_time:.2f} seconds')
utils.show_logs("Training loss", logs_train)
utils.show_logs("Validation loss", logs_val)
print('_'*50)
print('')
if logs_val["locAcc_val"] > best_acc:
best_state = deepcopy(fl.get_module(feature_maker).state_dict())
best_acc = logs_val["locAcc_val"]
logs["epoch"].append(epoch)
for key, value in dict(logs_train, **logs_val).items():
if key not in logs:
logs[key] = [None for x in range(epoch)]
if isinstance(value, np.ndarray):
value = value.tolist()
logs[key].append(value)
if (epoch % logs["saveStep"] == 0 and epoch > 0) or epoch == n_epochs - 1:
model_state_dict = fl.get_module(feature_maker).state_dict()
criterion_state_dict = fl.get_module(criterion).state_dict()
fl.save_checkpoint(model_state_dict, criterion_state_dict,
optimizer.state_dict(), best_state,
f"{path_checkpoint}_{epoch}.pt")
utils.save_logs(logs, f"{path_checkpoint}_logs.json")
logs = {"epoch": []}
optimizer = torch.optim.Adam(list(clf.parameters()),
lr=2e-3,
betas=(0.9, 0.999),
eps=2e-8)
best_acc = -1
start_time = time.time()
for ep in range(MAX_EPOCHS):
logs_train = train_step(feat_gen, clf, train_loader, optimizer, ep)
#logs_val = val_step(feature_maker, criterion, val_loader)
#logs = {"epoch": [], "iter": [], "saveStep": -1} # saveStep=-1, save only best checkpoint!
logs_test = test_step(feat_gen, clf, test_loader, optimizer, ep)
print('')
print('_' * 50)
print(f'Ran {ep + 1} epochs ' f'in {time.time() - start_time:.2f} seconds')
utils.show_logs("Training loss", logs_train)
utils.show_logs("Test loss", logs_test)
print('_' * 50)
print('')
if logs_test["locAcc_test"] > best_acc:
best_state = deepcopy(fl.get_module(feat_gen).state_dict())
best_acc = logs_test["locAcc_test"]
logs["epoch"].append(ep)
for key, value in dict(logs_train, **logs_test).items():
if key not in logs:
logs[key] = [None for x in range(ep)]
if isinstance(value, np.ndarray):
value = value.tolist()
logs[key].append(value)
def trainLinsepClassification(
feature_maker,
criterion, # combined with classification model before
train_loader,
val_loader,
optimizer,
path_logs,
logs_save_step,
path_best_checkpoint,
n_epochs,
cpc_epoch,
label_key="speaker",
centerpushSettings=None):
wasOptimizeCPC = feature_maker.optimize if hasattr(feature_maker, 'optimize') else None
feature_maker.eval()
feature_maker.optimize = False
start_epoch = 0
best_train_acc = -1
best_acc = -1
bect_epoch = -1
logs = {"epoch": [], "iter": [], "saveStep": logs_save_step}
start_time = time.time()
for epoch in range(start_epoch, n_epochs):
logs_train = train_step(feature_maker, criterion, train_loader,
optimizer, label_key, centerpushSettings=centerpushSettings)
logs_val = val_step(feature_maker, criterion, val_loader, label_key, centerpushSettings=centerpushSettings)
print('')
print('_'*50)
print(f'Ran {epoch + 1} {label_key} classification epochs '
f'in {time.time() - start_time:.2f} seconds')
utils.show_logs("Training loss", logs_train)
utils.show_logs("Validation loss", logs_val)
print('_'*50)
print('')
if logs_val["locAcc_val"] > best_acc:
best_state_cpc = deepcopy(fl.get_module(feature_maker).state_dict())
best_state_classif_crit = deepcopy(fl.get_module(criterion).state_dict())
optimizer_state_best_ep = optimizer.state_dict()
best_epoch = epoch
best_acc = logs_val["locAcc_val"]
if logs_train["locAcc_train"] > best_train_acc:
best_train_acc = logs_train["locAcc_train"]
logs["epoch"].append(epoch)
for key, value in dict(logs_train, **logs_val).items():
if key not in logs:
logs[key] = [None for x in range(epoch)]
if isinstance(value, np.ndarray):
value = value.tolist()
logs[key].append(value)
if (epoch % logs["saveStep"] == 0 and epoch > 0) or epoch == n_epochs - 1:
model_state_dict = fl.get_module(feature_maker).state_dict()
criterion_state_dict = fl.get_module(criterion).state_dict()
# fl.save_checkpoint(model_state_dict, criterion_state_dict,
# optimizer.state_dict(), best_state,
# f"{path_checkpoint}_{epoch}.pt")
utils.save_logs(logs, f"{path_logs}_logs.json")
if path_best_checkpoint:
save_linsep_best_checkpoint(best_state_cpc, best_state_classif_crit,
optimizer_state_best_ep, # TODO check if should save that epoch or last in optimizer
os.path.join(path_best_checkpoint, f"{label_key}_classif_best-epoch{best_epoch}-cpc_epoch{cpc_epoch}.pt"))
feature_maker.optimize = wasOptimizeCPC
return {'num_epoch_trained': n_epochs,
'best_val_acc': best_acc,
'best_train_acc': best_train_acc
}
