def trainStep(dataLoader,
cpcModel,
cpcCriterion,
optimizer,
scheduler,
loggingStep):
cpcModel.train()
cpcCriterion.train()
start_time = time.perf_counter()
n_examples = 0
logs, lastlogs = {}, None
iter = 0
for step, fulldata in enumerate(dataLoader):
batchData, labelData = fulldata
label = labelData['speaker']
n_examples += batchData.size(0)
batchData = batchData.cuda(non_blocking=True)
label = label.cuda(non_blocking=True)
c_feature, encoded_data, label = cpcModel(batchData, label)
allLosses, allAcc, _ = cpcCriterion(c_feature, encoded_data, label, None)
totLoss = allLosses.sum()
totLoss.backward()
# Show grads ?
optimizer.step()
optimizer.zero_grad()
if "locLoss_train" not in logs:
logs["locLoss_train"] = np.zeros(allLosses.size(1))
logs["locAcc_train"] = np.zeros(allLosses.size(1))
iter += 1
logs["locLoss_train"] += (allLosses.mean(dim=0)).detach().cpu().numpy()
logs["locAcc_train"] += (allAcc.mean(dim=0)).cpu().numpy()
if (step + 1) % loggingStep == 0:
new_time = time.perf_counter()
elapsed = new_time - start_time
print(f"Update {step + 1}")
print(f"elapsed: {elapsed:.1f} s")
print(
f"{1000.0 * elapsed / loggingStep:.1f} ms per batch, {1000.0 * elapsed / n_examples:.1f} ms / example")
locLogs = utils.update_logs(logs, loggingStep, lastlogs)
lastlogs = deepcopy(logs)
utils.show_logs("Training loss", locLogs)
start_time, n_examples = new_time, 0
if scheduler is not None:
scheduler.step()
logs = utils.update_logs(logs, iter)
logs["iter"] = iter
utils.show_logs("Average training loss on epoch", logs)
return logs
def train_step(feat_gen, clf, train_loader, optimizer, ep='unknown'):
feat_gen.eval()
clf.train()
#loss_func = nn.CrossEntropyLoss()
logs = {"locLoss_train": 0, "locAcc_train": 0}
for i, (batch_data, labels) in enumerate(train_loader):
#if i>3: break; # batch_data:(B,1,20480), labels:(B)
c, z, _ = feat_gen(batch_data,
None) # c:(B,128,256), z:(B:128,256) (B,T,F)
c = c.detach()
z = z.detach()
#c = c[:,-1, :]
# Predict --> loss
optimizer.zero_grad()
#logit = clf(c) # logit: (B,Class)
#loss = loss_func(logit, labels.cuda()).view(1, -1)
all_losses, all_acc = clf(c, None, labels)
totLoss = all_losses.sum()
totLoss.backward()
optimizer.step()
#acc = (logit.detach().cpu().max(1)[1] == labels).double().mean().view(1, -1)
logs["locLoss_train"] += np.asarray([all_losses.mean().item()])
logs["locAcc_train"] += np.asarray([all_acc.mean().item()])
logs = utils.update_logs(logs, i)
logs["iter"] = i
return logs
def train_step(feature_maker, criterion, data_loader, optimizer, label_key="speaker", centerpushSettings=None):
if feature_maker.optimize:
feature_maker.train()
criterion.train()
logs = {"locLoss_train": 0, "locAcc_train": 0}
for step, fulldata in enumerate(data_loader):
optimizer.zero_grad()
batch_data, label_data = fulldata
label = label_data[label_key]
c_feature, encoded_data, _ = feature_maker(batch_data, None)
if not feature_maker.optimize:
c_feature, encoded_data = c_feature.detach(), encoded_data.detach()
if centerpushSettings:
centers, pushDeg = centerpushSettings
c_feature = utils.pushToClosestForBatch(c_feature, centers, deg=pushDeg)
encoded_data = utils.pushToClosestForBatch(encoded_data, centers, deg=pushDeg)
all_losses, all_acc = criterion(c_feature, encoded_data, label)
totLoss = all_losses.sum()
totLoss.backward()
optimizer.step()
logs["locLoss_train"] += np.asarray([all_losses.mean().item()])
logs["locAcc_train"] += np.asarray([all_acc.mean().item()])
logs = utils.update_logs(logs, step)
logs["iter"] = step
return logs
def train_step(feature_maker, criterion, data_loader, optimizer):
if feature_maker.optimize:
feature_maker.train()
criterion.train()
logs = {"locLoss_train": 0, "locAcc_train": 0}
for step, fulldata in enumerate(data_loader):
optimizer.zero_grad()
batch_data, label = fulldata
c_feature, encoded_data, _ = feature_maker(batch_data, None)
if not feature_maker.optimize:
c_feature, encoded_data = c_feature.detach(), encoded_data.detach()
all_losses, all_acc = criterion(c_feature, encoded_data, label)
totLoss = all_losses.sum()
totLoss.backward()
optimizer.step()
logs["locLoss_train"] += np.asarray([all_losses.mean().item()])
logs["locAcc_train"] += np.asarray([all_acc.mean().item()])
logs = utils.update_logs(logs, step)
logs["iter"] = step
return logs
def valStep(dataLoader,
cpcModel,
cpcCriterion):
cpcCriterion.eval()
cpcModel.eval()
logs = {}
cpcCriterion.eval()
cpcModel.eval()
iter = 0
for step, fulldata in enumerate(dataLoader):
batchData, labelData = fulldata
label = labelData['speaker']
batchData = batchData.cuda(non_blocking=True)
label = label.cuda(non_blocking=True)
with torch.no_grad():
c_feature, encoded_data, label = cpcModel(batchData, label)
allLosses, allAcc, _ = cpcCriterion(c_feature, encoded_data, label, None)
if "locLoss_val" not in logs:
logs["locLoss_val"] = np.zeros(allLosses.size(1))
logs["locAcc_val"] = np.zeros(allLosses.size(1))
iter += 1
logs["locLoss_val"] += allLosses.mean(dim=0).cpu().numpy()
logs["locAcc_val"] += allAcc.mean(dim=0).cpu().numpy()
logs = utils.update_logs(logs, iter)
logs["iter"] = iter
utils.show_logs("Validation loss:", logs)
return logs
def val_step(feature_maker, criterion, data_loader):
feature_maker.eval()
criterion.eval()
logs = {"locLoss_val": 0, "locAcc_val": 0}
for step, fulldata in enumerate(data_loader):
with torch.no_grad():
batch_data, label = fulldata
c_feature, encoded_data, _ = feature_maker(batch_data, None)
all_losses, all_acc = criterion(c_feature, encoded_data, label)
logs["locLoss_val"] += np.asarray([all_losses.mean().item()])
logs["locAcc_val"] += np.asarray([all_acc.mean().item()])
logs = utils.update_logs(logs, step)
return logs
def test_step(feat_gen, clf, test_loader, optimizer, ep='unknown'):
feat_gen.eval()
clf.eval()
logs = {"locLoss_test": 0, "locAcc_test": 0}
for i, (batch_data, labels) in enumerate(train_loader):
#if i>3: break; # batch_data:(B,1,20480), labels:(B)
c, z, _ = feat_gen(batch_data,
None) # c:(B,128,256), z:(B:128,256) (B,T,F)
c = c.detach()
z = z.detach()
# Predict --> loss
all_losses, all_acc = clf(c, None, labels)
logs["locLoss_test"] += np.asarray([all_losses.mean().item()])
logs["locAcc_test"] += np.asarray([all_acc.mean().item()])
logs = utils.update_logs(logs, i)
logs["iter"] = i
return logs
def val_step(feature_maker, criterion, data_loader, label_key="speaker", centerpushSettings=None):
feature_maker.eval()
criterion.eval()
logs = {"locLoss_val": 0, "locAcc_val": 0}
for step, fulldata in enumerate(data_loader):
with torch.no_grad():
batch_data, label_data = fulldata
label = label_data[label_key]
c_feature, encoded_data, _ = feature_maker(batch_data, None)
if centerpushSettings:
centers, pushDeg = centerpushSettings
c_feature = utils.pushToClosestForBatch(c_feature, centers, deg=pushDeg)
encoded_data = utils.pushToClosestForBatch(encoded_data, centers, deg=pushDeg)
all_losses, all_acc = criterion(c_feature, encoded_data, label)
logs["locLoss_val"] += np.asarray([all_losses.mean().item()])
logs["locAcc_val"] += np.asarray([all_acc.mean().item()])
logs = utils.update_logs(logs, step)
return logs
