def main(config):
testIvectorDataset = TestIvectorDataset(config)
pSVM = PSVM(testIvectorDataset.ivecs)
pSVM.load_model(config.ipath2model)
pSVM.scoring(testIvectorDataset, testIvectorDataset.idx2utt,
config.opath2score)
compute_eer(config.opath2score, config.trials)
def train(self, n_epochs=1, save_every=1):
while (self.cur_epoch < n_epochs):
print('Epoch {}/{}'.format(self.cur_epoch + 1, n_epochs))
train_iter = tqdm(enumerate(self.train_loader))
train_loss_epoch = 0.0
for t, batch in train_iter:
train_loss = self.train_step(batch)
self.history['train_loss_batch'].append(train_loss)
train_loss_epoch += train_loss
self.total_iters += 1
self.history['train_loss'].append(train_loss_epoch / (t + 1))
print('Total train loss: {:0.4f}'.format(
self.history['train_loss'][-1]))
if self.valid_loader is not None:
scores, labels = None, None
for t, batch in enumerate(self.valid_loader):
scores_batch, labels_batch = self.valid(batch)
try:
scores = np.concatenate([scores, scores_batch], 0)
labels = np.concatenate([labels, labels_batch], 0)
except:
scores, labels = scores_batch, labels_batch
self.history['valid_loss'].append(compute_eer(labels, scores))
print(
'Current validation loss, best validation loss, and epoch: {:0.4f}, {:0.4f}, {}'
.format(self.history['valid_loss'][-1],
np.min(self.history['valid_loss']),
1 + np.argmin(self.history['valid_loss'])))
self.scheduler.step()
print('Current LR: {}'.format(
self.optimizer.param_groups[0]['lr']))
self.cur_epoch += 1
if self.cur_epoch % save_every == 0 or self.history['valid_loss'][
-1] < np.min([np.inf] + self.history['valid_loss'][:-1]):
self.checkpointing()
print('Training done!')
if self.valid_loader is not None:
print('Best validation loss and corresponding epoch: {:0.4f}, {}'.
format(np.min(self.history['valid_loss']),
1 + np.argmin(self.history['valid_loss'])))
return np.min(self.history['valid_loss'])
def validate_verification(cfg, model, test_loader):
batch_time = AverageMeter('Time', ':6.3f')
progress = ProgressMeter(len(test_loader),
batch_time,
prefix='Test: ',
logger=logger)
# switch to evaluate mode
model.eval()
labels, distances = [], []
with torch.no_grad():
end = time.time()
for i, (input1, input2, label) in enumerate(test_loader):
input1 = input1.cuda(non_blocking=True).squeeze(0)
input2 = input2.cuda(non_blocking=True).squeeze(0)
label = label.cuda(non_blocking=True)
# compute output
outputs1 = model(input1).mean(dim=0).unsqueeze(0)
outputs2 = model(input2).mean(dim=0).unsqueeze(0)
dists = F.cosine_similarity(outputs1, outputs2)
dists = dists.data.cpu().numpy()
distances.append(dists)
labels.append(label.data.cpu().numpy())
# measure elapsed time
batch_time.update(time.time() - end)
end = time.time()
if i % 2000 == 0:
progress.print(i)
labels = np.array([sublabel for label in labels for sublabel in label])
distances = np.array(
[subdist for dist in distances for subdist in dist])
eer = compute_eer(distances, labels)
logger.info('Test EER: {:.8f}'.format(np.mean(eer)))
return eer
def train(self, n_epochs=1, save_every=1):
while (self.cur_epoch < n_epochs):
np.random.seed()
if self.verbose>0:
print(' ')
print('Epoch {}/{}'.format(self.cur_epoch+1, n_epochs))
train_iter = tqdm(enumerate(self.train_loader))
else:
train_iter = enumerate(self.train_loader)
if self.pretrain:
ce_epoch=0.0
for t, batch in train_iter:
ce = self.pretrain_step(batch)
self.history['train_loss_batch'].append(ce)
ce_epoch+=ce
self.total_iters += 1
self.history['train_loss'].append(ce_epoch/(t+1))
if self.verbose>0:
print('Train loss: {:0.4f}'.format(self.history['train_loss'][-1]))
else:
train_loss_epoch=0.0
ce_loss_epoch=0.0
bin_loss_epoch=0.0
for t, batch in train_iter:
train_loss, ce_loss, bin_loss = self.train_step(batch)
self.history['train_loss_batch'].append(train_loss)
self.history['ce_loss_batch'].append(ce_loss)
self.history['bin_loss_batch'].append(bin_loss)
train_loss_epoch+=train_loss
ce_loss_epoch+=ce_loss
bin_loss_epoch+=bin_loss
self.total_iters += 1
self.history['train_loss'].append(train_loss_epoch/(t+1))
self.history['ce_loss'].append(ce_loss_epoch/(t+1))
self.history['bin_loss'].append(bin_loss_epoch/(t+1))
if self.verbose>0:
print(' ')
print('Total train loss: {:0.4f}'.format(self.history['train_loss'][-1]))
print('CE loss: {:0.4f}'.format(self.history['ce_loss'][-1]))
print('Binary classification loss: {:0.4f}'.format(self.history['bin_loss'][-1]))
print(' ')
if self.valid_loader is not None:
tot_correct, tot_ = 0, 0
e2e_scores, cos_scores, labels = None, None, None
for t, batch in enumerate(self.valid_loader):
correct, total, e2e_scores_batch, cos_scores_batch, labels_batch = self.valid(batch)
try:
e2e_scores = np.concatenate([e2e_scores, e2e_scores_batch], 0)
cos_scores = np.concatenate([cos_scores, cos_scores_batch], 0)
labels = np.concatenate([labels, labels_batch], 0)
except:
e2e_scores, cos_scores, labels = e2e_scores_batch, cos_scores_batch, labels_batch
tot_correct += correct
tot_ += total
self.history['e2e_eer'].append(compute_eer(labels, e2e_scores))
self.history['cos_eer'].append(compute_eer(labels, cos_scores))
self.history['ErrorRate'].append(1.-float(tot_correct)/tot_)
if self.verbose>0:
print(' ')
print('Current e2e EER, best e2e EER, and epoch: {:0.4f}, {:0.4f}, {}'.format(self.history['e2e_eer'][-1], np.min(self.history['e2e_eer']), 1+np.argmin(self.history['e2e_eer'])))
print('Current cos EER, best cos EER, and epoch: {:0.4f}, {:0.4f}, {}'.format(self.history['cos_eer'][-1], np.min(self.history['cos_eer']), 1+np.argmin(self.history['cos_eer'])))
print('Current Error rate, best Error rate, and epoch: {:0.4f}, {:0.4f}, {}'.format(self.history['ErrorRate'][-1], np.min(self.history['ErrorRate']), 1+np.argmin(self.history['ErrorRate'])))
self.scheduler.step(np.min([self.history['e2e_eer'][-1], self.history['cos_eer'][-1]]))
else:
self.scheduler.step()
if self.verbose>0:
print('Current LR: {}'.format(self.optimizer.param_groups[0]['lr']))
self.cur_epoch += 1
if self.valid_loader is not None and self.save_cp and (self.cur_epoch % save_every == 0 or self.history['e2e_eer'][-1] < np.min([np.inf]+self.history['e2e_eer'][:-1]) or self.history['cos_eer'][-1] < np.min([np.inf]+self.history['cos_eer'][:-1])):
self.checkpointing()
elif self.save_cp and self.cur_epoch % save_every == 0:
self.checkpointing()
if self.verbose>0:
print('Training done!')
if self.valid_loader is not None:
if self.verbose>0:
print('Best e2e eer and corresponding epoch: {:0.4f}, {}'.format(np.min(self.history['e2e_eer']), 1+np.argmin(self.history['e2e_eer'])))
print('Best cos eer and corresponding epoch: {:0.4f}, {}'.format(np.min(self.history['cos_eer']), 1+np.argmin(self.history['cos_eer'])))
return [np.min(self.history['e2e_eer']), np.min(self.history['cos_eer']), np.min(self.history['ErrorRate'])]
else:
return [np.min(self.history['train_loss'])]
def train(self, n_epochs=1, save_every=1):
while self.cur_epoch < n_epochs:
np.random.seed()
if self.verbose > 0:
print(' ')
print('Epoch {}/{}'.format(self.cur_epoch + 1, n_epochs))
train_iter = tqdm(enumerate(self.train_loader))
else:
train_iter = enumerate(self.train_loader)
ce = 0.0
triplet_loss = 0.0
train_loss = 0.0
# Train step
for t, batch in train_iter:
ce_batch, triplet_loss_batch = self.train_step(batch)
ce += ce_batch
triplet_loss += triplet_loss_batch
train_loss += ce_batch + triplet_loss_batch
self.history['train_loss_batch'].append(ce_batch +
triplet_loss_batch)
self.history['triplet_loss_batch'].append(triplet_loss_batch)
self.history['ce_loss_batch'].append(ce_batch)
self.total_iters += 1
self.history['train_loss'].append(train_loss / (t + 1))
self.history['triplet_loss'].append(triplet_loss / (t + 1))
self.history['ce_loss'].append(ce / (t + 1))
if self.verbose > 0:
print(' ')
print(
'Total train loss, Triplet loss, and Cross-entropy: {:0.4f}, {:0.4f}, {:0.4f}'
.format(self.history['train_loss'][-1],
self.history['triplet_loss'][-1],
self.history['ce_loss'][-1]))
# Validation
tot_correct = 0
tot_ = 0
scores, labels = None, None
for t, batch in enumerate(self.valid_loader):
correct, total, scores_batch, labels_batch = self.valid(batch)
try:
scores = np.concatenate([scores, scores_batch], 0)
labels = np.concatenate([labels, labels_batch], 0)
except:
scores, labels = scores_batch, labels_batch
tot_correct += correct
tot_ += total
self.history['EER'].append(compute_eer(labels, scores))
self.history['ErrorRate'].append(1. - float(tot_correct) / tot_)
if self.verbose > 0:
print(' ')
print(
'Current, best validation error rate, and epoch: {:0.4f}, {:0.4f}, {}'
.format(self.history['ErrorRate'][-1],
np.min(self.history['ErrorRate']),
1 + np.argmin(self.history['ErrorRate'])))
print(' ')
print(
'Current, best validation EER, and epoch: {:0.4f}, {:0.4f}, {}'
.format(self.history['EER'][-1],
np.min(self.history['EER']),
1 + np.argmin(self.history['EER'])))
self.scheduler.step(self.history['ErrorRate'][-1])
if self.verbose > 0:
print(' ')
print('Current LR: {}'.format(
self.optimizer.param_groups[0]['lr']))
if self.save_cp and (
self.cur_epoch % save_every == 0 or
(self.history['ErrorRate'][-1] <
np.min([np.inf] + self.history['ErrorRate'][:-1])) or
(self.history['EER'][-1] <
np.min([np.inf] + self.history['EER'][:-1]))):
self.checkpointing()
self.cur_epoch += 1
if self.verbose > 0:
print('Training done!')
if self.valid_loader is not None:
print('Best error rate and corresponding epoch: {:0.4f}, {}'.
format(np.min(self.history['ErrorRate']),
1 + np.argmin(self.history['ErrorRate'])))
print('Best EER and corresponding epoch: {:0.4f}, {}'.format(
np.min(self.history['EER']),
1 + np.argmin(self.history['EER'])))
return np.min(self.history['ErrorRate'])
def train(self, n_epochs=1, save_every=1):
while self.cur_epoch < n_epochs:
np.random.seed()
if isinstance(self.train_loader.dataset, Loader):
self.train_loader.dataset.update_lists()
adjust_learning_rate(self.optimizer, self.cur_epoch, self.base_lr)
if self.verbose > 0:
print(' ')
print('Epoch {}/{}'.format(self.cur_epoch + 1, n_epochs))
train_iter = tqdm(enumerate(self.train_loader))
else:
train_iter = enumerate(self.train_loader)
ce = 0.0
triplet_loss = 0.0
train_loss = 0.0
# Train step
for t, batch in train_iter:
ce_batch, triplet_loss_batch = self.train_step(batch)
ce += ce_batch
triplet_loss += triplet_loss_batch
train_loss += ce_batch + triplet_loss_batch
self.history['train_loss_batch'].append(ce_batch +
triplet_loss_batch)
self.history['triplet_loss_batch'].append(triplet_loss_batch)
self.history['ce_loss_batch'].append(ce_batch)
self.total_iters += 1
self.history['train_loss'].append(train_loss / (t + 1))
self.history['triplet_loss'].append(triplet_loss / (t + 1))
self.history['ce_loss'].append(ce / (t + 1))
if self.verbose > 0:
print(' ')
print(
'Total train loss, Triplet loss, and Cross-entropy: {:0.4f}, {:0.4f}, {:0.4f}'
.format(self.history['train_loss'][-1],
self.history['triplet_loss'][-1],
self.history['ce_loss'][-1]))
# Validation
tot_correct_1, tot_correct_5, tot_ = 0, 0, 0
scores, labels = None, None
for t, batch in enumerate(self.valid_loader):
correct_1, correct_5, total, scores_batch, labels_batch = self.valid(
batch)
try:
scores = np.concatenate([scores, scores_batch], 0)
labels = np.concatenate([labels, labels_batch], 0)
except:
scores, labels = scores_batch, labels_batch
tot_correct_1 += correct_1
tot_correct_5 += correct_5
tot_ += total
self.history['EER'].append(compute_eer(labels, scores))
self.history['acc_1'].append(float(tot_correct_1) / tot_)
self.history['acc_5'].append(float(tot_correct_5) / tot_)
if self.verbose > 0:
print(' ')
print(
'Current, best validation EER, and epoch: {:0.4f}, {:0.4f}, {}'
.format(self.history['EER'][-1],
np.min(self.history['EER']),
1 + np.argmin(self.history['EER'])))
print(
'Current Top 1 Acc, best Top 1 Acc, and epoch: {:0.4f}, {:0.4f}, {}'
.format(self.history['acc_1'][-1],
np.max(self.history['acc_1']),
1 + np.argmax(self.history['acc_1'])))
print(
'Current Top 5 Acc, best Top 5 Acc, and epoch: {:0.4f}, {:0.4f}, {}'
.format(self.history['acc_5'][-1],
np.max(self.history['acc_5']),
1 + np.argmax(self.history['acc_5'])))
if self.verbose > 0:
print(' ')
print('Current LR: {}'.format(
self.optimizer.param_groups[0]['lr']))
if self.save_cp and (self.cur_epoch % save_every == 0
or self.history['EER'][-1] <
np.min([np.inf] + self.history['EER'][:-1])):
self.checkpointing()
self.cur_epoch += 1
if self.verbose > 0:
print('Training done!')
if self.valid_loader is not None:
print('Best EER and corresponding epoch: {:0.4f}, {}'.format(
np.min(self.history['EER']),
1 + np.argmin(self.history['EER'])))
return np.min(self.history['EER'])
def train(self, n_epochs=1, save_every=1):
while (self.cur_epoch < n_epochs):
np.random.seed()
if self.verbose > 0:
print(' ')
print('Epoch {}/{}'.format(self.cur_epoch + 1, n_epochs))
train_iter = tqdm(enumerate(self.train_loader))
else:
train_iter = enumerate(self.train_loader)
ce_asv_epoch = 0.0
ce_lid_epoch = 0.0
for t, batch in train_iter:
ce_asv, ce_lid = self.train_step(batch)
self.history['ce_asv_batch'].append(ce_asv)
self.history['ce_lid_batch'].append(ce_lid)
ce_asv_epoch += ce_asv
ce_lid_epoch += ce_lid
self.total_iters += 1
self.history['ce_asv'].append(ce_asv_epoch / (t + 1))
self.history['ce_lid'].append(ce_lid_epoch / (t + 1))
if self.verbose > 0:
print(
'Total train loss, ASV CE, and LID CE: {:0.4f}, {:0.4f}, {:0.4f}'
.format(
self.history['ce_asv'][-1] +
self.history['ce_lid'][-1], self.history['ce_asv'][-1],
self.history['ce_lid'][-1]))
scores, labels = None, None
for t, batch in enumerate(self.valid_loader):
scores_batch, labels_batch = self.valid(batch)
try:
scores = np.concatenate([scores, scores_batch], 0)
labels = np.concatenate([labels, labels_batch], 0)
except:
scores, labels = scores_batch, labels_batch
self.history['valid_loss'].append(compute_eer(labels, scores))
if self.verbose > 0:
print(
'Current validation loss, best validation loss, and epoch: {:0.4f}, {:0.4f}, {}'
.format(self.history['valid_loss'][-1],
np.min(self.history['valid_loss']),
1 + np.argmin(self.history['valid_loss'])))
self.scheduler_s.step(self.history['valid_loss'][-1])
self.scheduler_l.step(self.history['valid_loss'][-1])
if self.verbose > 0:
print('Current LR: {}'.format(
self.optimizer.param_groups[0]['lr']))
self.cur_epoch += 1
if self.save_cp and (
self.cur_epoch % save_every == 0
or self.history['valid_loss'][-1] <
np.min([np.inf] + self.history['valid_loss'][:-1])):
self.checkpointing()
if self.verbose > 0:
print('Training done!')
if self.verbose > 0:
print('Best validation loss and corresponding epoch: {:0.4f}, {}'.
format(np.min(self.history['valid_loss']),
1 + np.argmin(self.history['valid_loss'])))
return np.min(self.history['valid_loss'])
def evaluate(self):
if self.verbose > 0:
print('\nIteration - Epoch: {} - {}'.format(
self.total_iters, self.cur_epoch))
e2e_scores, cos_scores, labels = None, None, None
for t, batch in enumerate(self.valid_loader):
e2e_scores_batch, cos_scores_batch, labels_batch = self.valid(
batch)
try:
e2e_scores = np.concatenate([e2e_scores, e2e_scores_batch], 0)
cos_scores = np.concatenate([cos_scores, cos_scores_batch], 0)
labels = np.concatenate([labels, labels_batch], 0)
except:
e2e_scores, cos_scores, labels = e2e_scores_batch, cos_scores_batch, labels_batch
self.history['e2e_eer'].append(compute_eer(labels, e2e_scores))
self.history['cos_eer'].append(compute_eer(labels, cos_scores))
if self.history['e2e_eer'][-1] < self.best_e2e_eer:
self.best_e2e_eer = self.history['e2e_eer'][-1]
self.best_e2e_eer_epoch = self.cur_epoch
self.best_e2e_eer_iteration = self.total_iters
if self.history['cos_eer'][-1] < self.best_cos_eer:
self.best_cos_eer = self.history['cos_eer'][-1]
self.best_cos_eer_epoch = self.cur_epoch
self.best_cos_eer_iteration = self.total_iters
if self.logger:
self.logger.add_scalar('Valid/E2E EER',
self.history['e2e_eer'][-1],
self.total_iters)
self.logger.add_scalar('Valid/Best E2E EER',
np.min(self.history['e2e_eer']),
self.total_iters)
self.logger.add_scalar('Valid/Cosine EER',
self.history['cos_eer'][-1],
self.total_iters)
self.logger.add_scalar('Valid/Best Cosine EER',
np.min(self.history['cos_eer']),
self.total_iters)
self.logger.add_pr_curve('E2E ROC',
labels=labels,
predictions=e2e_scores,
global_step=self.total_iters)
self.logger.add_pr_curve('Cosine ROC',
labels=labels,
predictions=cos_scores,
global_step=self.total_iters)
self.logger.add_histogram('Valid/COS_Scores',
values=cos_scores,
global_step=self.total_iters)
self.logger.add_histogram('Valid/E2E_Scores',
values=e2e_scores,
global_step=self.total_iters)
self.logger.add_histogram('Valid/Labels',
values=labels,
global_step=self.total_iters)
if self.verbose > 0:
print(' ')
print(
'Current e2e EER, best e2e EER, and epoch - iteration: {:0.4f}, {:0.4f}, {}, {}'
.format(self.history['e2e_eer'][-1],
np.min(self.history['e2e_eer']),
self.best_e2e_eer_epoch, self.best_e2e_eer_iteration))
print(
'Current cos EER, best cos EER, and epoch - iteration: {:0.4f}, {:0.4f}, {}, {}'
.format(self.history['cos_eer'][-1],
np.min(self.history['cos_eer']),
self.best_cos_eer_epoch, self.best_cos_eer_iteration))
def train(self, n_epochs=1, save_every=1):
while (self.cur_epoch < n_epochs):
np.random.seed()
if isinstance(self.train_loader.dataset, Loader):
self.train_loader.dataset.update_lists()
if self.verbose > 0:
print(' ')
print('Epoch {}/{}'.format(self.cur_epoch + 1, n_epochs))
train_iter = tqdm(enumerate(self.train_loader))
else:
train_iter = enumerate(self.train_loader)
if self.pretrain:
ce_epoch = 0.0
for t, batch in train_iter:
ce = self.pretrain_step(batch)
self.history['train_loss_batch'].append(ce)
ce_epoch += ce
if self.logger:
self.logger.add_scalar('Train/Cross entropy', ce,
self.total_iters)
self.logger.add_scalar(
'Info/LR', self.optimizer.param_groups[0]['lr'],
self.total_iters)
self.total_iters += 1
self.history['train_loss'].append(ce_epoch / (t + 1))
if self.verbose > 0:
print('Train loss: {:0.4f}'.format(
self.history['train_loss'][-1]))
else:
train_loss_epoch = 0.0
ce_loss_epoch = 0.0
bin_loss_epoch = 0.0
for t, batch in train_iter:
train_loss, ce_loss, bin_loss = self.train_step(batch)
self.history['train_loss_batch'].append(train_loss)
self.history['ce_loss_batch'].append(ce_loss)
self.history['bin_loss_batch'].append(bin_loss)
train_loss_epoch += train_loss
ce_loss_epoch += ce_loss
bin_loss_epoch += bin_loss
if self.logger:
self.logger.add_scalar('Train/Total train Loss',
train_loss, self.total_iters)
self.logger.add_scalar('Train/Binary class. Loss',
bin_loss, self.total_iters)
self.logger.add_scalar('Train/Cross enropy', ce_loss,
self.total_iters)
self.logger.add_scalar(
'Info/LR', self.optimizer.param_groups[0]['lr'],
self.total_iters)
self.total_iters += 1
self.history['train_loss'].append(train_loss_epoch / (t + 1))
self.history['ce_loss'].append(ce_loss_epoch / (t + 1))
self.history['bin_loss'].append(bin_loss_epoch / (t + 1))
if self.verbose > 0:
print(' ')
print('Total train loss: {:0.4f}'.format(
self.history['train_loss'][-1]))
print('CE loss: {:0.4f}'.format(
self.history['ce_loss'][-1]))
print('Binary classification loss: {:0.4f}'.format(
self.history['bin_loss'][-1]))
print(' ')
if self.valid_loader is not None:
e2e_scores, cos_scores, labels = None, None, None
for t, batch in enumerate(self.valid_loader):
e2e_scores_batch, cos_scores_batch, labels_batch = self.valid(
batch)
try:
e2e_scores = np.concatenate(
[e2e_scores, e2e_scores_batch], 0)
cos_scores = np.concatenate(
[cos_scores, cos_scores_batch], 0)
labels = np.concatenate([labels, labels_batch], 0)
except:
e2e_scores, cos_scores, labels = e2e_scores_batch, cos_scores_batch, labels_batch
self.history['e2e_eer'].append(compute_eer(labels, e2e_scores))
self.history['cos_eer'].append(compute_eer(labels, cos_scores))
if self.logger:
self.logger.add_scalar('Valid/E2E EER',
self.history['e2e_eer'][-1],
self.total_iters - 1)
self.logger.add_scalar('Valid/Best E2E EER',
np.min(self.history['e2e_eer']),
self.total_iters - 1)
self.logger.add_scalar('Valid/Cosine EER',
self.history['cos_eer'][-1],
self.total_iters - 1)
self.logger.add_scalar('Valid/Best Cosine EER',
np.min(self.history['cos_eer']),
self.total_iters - 1)
self.logger.add_pr_curve('E2E ROC',
labels=labels,
predictions=e2e_scores,
global_step=self.total_iters - 1)
self.logger.add_pr_curve('Cosine ROC',
labels=labels,
predictions=cos_scores,
global_step=self.total_iters - 1)
self.logger.add_histogram('Valid/COS_Scores',
values=cos_scores,
global_step=self.total_iters - 1)
self.logger.add_histogram('Valid/E2E_Scores',
values=e2e_scores,
global_step=self.total_iters - 1)
self.logger.add_histogram('Valid/Labels',
values=labels,
global_step=self.total_iters - 1)
if self.verbose > 0:
print(' ')
print(
'Current e2e EER, best e2e EER, and epoch: {:0.4f}, {:0.4f}, {}'
.format(self.history['e2e_eer'][-1],
np.min(self.history['e2e_eer']),
1 + np.argmin(self.history['e2e_eer'])))
print(
'Current cos EER, best cos EER, and epoch: {:0.4f}, {:0.4f}, {}'
.format(self.history['cos_eer'][-1],
np.min(self.history['cos_eer']),
1 + np.argmin(self.history['cos_eer'])))
self.scheduler.step(
np.min([
self.history['e2e_eer'][-1],
self.history['cos_eer'][-1]
]))
else:
self.scheduler.step()
if self.verbose > 0:
print('Current LR: {}'.format(
self.optimizer.param_groups[0]['lr']))
self.cur_epoch += 1
if self.valid_loader is not None and self.save_cp and (
self.cur_epoch % save_every == 0
or self.history['e2e_eer'][-1] <
np.min([np.inf] + self.history['e2e_eer'][:-1])
or self.history['cos_eer'][-1] <
np.min([np.inf] + self.history['cos_eer'][:-1])):
self.checkpointing()
elif self.save_cp and self.cur_epoch % save_every == 0:
self.checkpointing()
if self.verbose > 0:
print('Training done!')
if self.valid_loader is not None:
if self.verbose > 0:
print(
'Best e2e eer and corresponding epoch: {:0.4f}, {}'.format(
np.min(self.history['e2e_eer']),
1 + np.argmin(self.history['e2e_eer'])))
print(
'Best cos eer and corresponding epoch: {:0.4f}, {}'.format(
np.min(self.history['cos_eer']),
1 + np.argmin(self.history['cos_eer'])))
return [
np.min(self.history['e2e_eer']),
np.min(self.history['cos_eer'])
]
else:
return [np.min(self.history['train_loss'])]
def train(self, n_epochs=1, save_every=1):
while (self.cur_epoch < n_epochs):
if self.verbose > 0:
print(' ')
print('Epoch {}/{}'.format(self.cur_epoch + 1, n_epochs))
train_iter = tqdm(enumerate(self.train_loader),
total=len(self.train_loader))
else:
train_iter = enumerate(self.train_loader)
train_loss_epoch = 0.0
for t, batch in train_iter:
train_loss = self.train_step(batch)
self.history['train_loss_batch'].append(train_loss)
train_loss_epoch += train_loss
if self.logger:
self.logger.add_scalar('Train/Train Loss', train_loss,
self.total_iters)
self.logger.add_scalar(
'Info/LR',
self.optimizer.optimizer.param_groups[0]['lr'],
self.total_iters)
self.logger.add_scalar('Info/Epoch', self.cur_epoch,
self.total_iters)
self.total_iters += 1
self.history['train_loss'].append(train_loss_epoch / (t + 1))
if self.verbose > 0:
print('Total train loss: {:0.4f}'.format(
self.history['train_loss'][-1]))
if self.valid_loader is not None:
scores, labels = None, None
for t, batch in enumerate(self.valid_loader):
scores_batch, labels_batch = self.valid(batch)
try:
scores = np.concatenate([scores, scores_batch], 0)
labels = np.concatenate([labels, labels_batch], 0)
except:
scores, labels = scores_batch, labels_batch
self.history['valid_loss'].append(compute_eer(labels, scores))
if self.logger:
self.logger.add_scalar('Valid/Valid EER',
self.history['valid_loss'][-1],
self.total_iters)
self.logger.add_scalar('Valid/Best valid EER',
np.min(self.history['valid_loss']),
self.total_iters)
self.logger.add_pr_curve('E2E ROC',
labels=labels,
predictions=scores,
global_step=self.total_iters)
self.logger.add_histogram('Valid/Scores',
values=scores,
global_step=self.total_iters - 1)
self.logger.add_histogram('Valid/Labels',
values=labels,
global_step=self.total_iters - 1)
if self.verbose > 0:
print(
'Current validation loss, best validation loss, and epoch: {:0.4f}, {:0.4f}, {}'
.format(self.history['valid_loss'][-1],
np.min(self.history['valid_loss']),
1 + np.argmin(self.history['valid_loss'])))
if self.verbose > 0:
print('Current LR: {}'.format(
self.optimizer.optimizer.param_groups[0]['lr']))
self.cur_epoch += 1
if (self.cur_epoch % save_every == 0
or self.history['valid_loss'][-1] <
np.min([np.inf] + self.history['valid_loss'][:-1])
) and self.save_cp:
self.checkpointing()
if self.verbose > 0:
print('Training done!')
if self.valid_loader is not None:
if self.verbose > 0:
print(
'Best validation loss and corresponding epoch: {:0.4f}, {}'
.format(np.min(self.history['valid_loss']),
1 + np.argmin(self.history['valid_loss'])))
return np.min(self.history['valid_loss'])
def train(self, n_epochs=1, save_every=1):
while (self.cur_epoch < n_epochs):
print('Epoch {}/{}'.format(self.cur_epoch + 1, n_epochs))
train_iter = tqdm(enumerate(self.train_loader))
train_loss_epoch = 0.0
train_all_epoch = 0.0
train_la_epoch = 0.0
train_pa_epoch = 0.0
train_mix_epoch = 0.0
for t, batch in train_iter:
train_loss, train_all, train_la, train_pa, train_mix = self.train_step(
batch)
self.history['train_loss_batch'].append(train_loss)
self.history['train_all_batch'].append(train_all)
self.history['train_la_batch'].append(train_la)
self.history['train_pa_batch'].append(train_pa)
self.history['train_mix_batch'].append(train_mix)
train_loss_epoch += train_loss
train_all_epoch += train_all
train_la_epoch += train_la
train_pa_epoch += train_pa
train_mix_epoch += train_mix
if self.logger:
self.logger.add_scalar('Train/Total train loss',
train_loss, self.total_iters)
self.logger.add_scalar('Train/Train Loss mixture',
train_all, self.total_iters)
self.logger.add_scalar('Train/Train Loss LA', train_la,
self.total_iters)
self.logger.add_scalar('Train/Train Loss PA', train_pa,
self.total_iters)
self.logger.add_scalar('Train/Train Loss MIX', train_mix,
self.total_iters)
self.logger.add_scalar(
'Info/LR_LA',
self.optimizer_la.optimizer.param_groups[0]['lr'],
self.total_iters)
self.logger.add_scalar(
'Info/LR_PA',
self.optimizer_pa.optimizer.param_groups[0]['lr'],
self.total_iters)
self.logger.add_scalar(
'Info/LR_MIX',
self.optimizer_mix.optimizer.param_groups[0]['lr'],
self.total_iters)
self.total_iters += 1
self.history['train_loss'].append(train_loss_epoch / (t + 1))
self.history['train_all'].append(train_all_epoch / (t + 1))
self.history['train_la'].append(train_la_epoch / (t + 1))
self.history['train_pa'].append(train_pa_epoch / (t + 1))
self.history['train_mix'].append(train_mix_epoch / (t + 1))
print(
'Total train loss, loss_all, loss_la, loss_pa, loss_mix: {:0.4f}, {:0.4f}, {:0.4f}, {:0.4f}, {:0.4f}'
.format(self.history['train_loss'][-1],
self.history['train_all'][-1],
self.history['train_la'][-1],
self.history['train_pa'][-1],
self.history['train_mix'][-1]))
scores_all, scores_la, scores_pa, scores_mix, labels = None, None, None, None, None
if self.train_mode == 'mix':
for t, batch in enumerate(self.valid_loader):
scores_all_batch, scores_la_batch, scores_pa_batch, scores_mix_batch, labels_batch = self.valid(
batch)
try:
scores_all = np.concatenate(
[scores_all, scores_all_batch], 0)
scores_la = np.concatenate(
[scores_la, scores_la_batch], 0)
scores_pa = np.concatenate(
[scores_pa, scores_pa_batch], 0)
scores_mix = np.concatenate(
[scores_mix, scores_mix_batch], 0)
labels = np.concatenate([labels, labels_batch], 0)
except:
scores_all, scores_la, scores_pa, scores_mix, labels = scores_all_batch, scores_la_batch, scores_pa_batch, scores_mix_batch, labels_batch
self.history['valid_all'].append(
compute_eer(labels, scores_all))
self.history['valid_la'].append(compute_eer(labels, scores_la))
self.history['valid_pa'].append(compute_eer(labels, scores_pa))
self.history['valid_mix'].append(
compute_eer(labels, scores_mix))
if self.logger:
self.logger.add_scalar('Valid/Valid EER mixture',
self.history['valid_all'][-1],
self.total_iters - 1)
self.logger.add_scalar('Valid/Best valid EER mixture',
np.min(self.history['valid_all']),
self.total_iters - 1)
self.logger.add_pr_curve('Valid. ROC mixture',
labels=labels,
predictions=scores_all,
global_step=self.total_iters - 1)
self.logger.add_histogram('Valid/Scores_Mixture',
values=scores_all,
global_step=self.total_iters - 1)
print(
'ALL: Current validation loss, best validation loss, and epoch: {:0.4f}, {:0.4f}, {}'
.format(self.history['valid_all'][-1],
np.min(self.history['valid_all']),
1 + np.argmin(self.history['valid_all'])))
else:
for t, batch in enumerate(self.valid_loader):
scores_la_batch, scores_pa_batch, scores_mix_batch, labels_batch = self.valid(
batch)
try:
scores_la = np.concatenate(
[scores_la, scores_la_batch], 0)
scores_pa = np.concatenate(
[scores_pa, scores_pa_batch], 0)
scores_mix = np.concatenate(
[scores_mix, scores_mix_batch], 0)
labels = np.concatenate([labels, labels_batch], 0)
except:
scores_la, scores_pa, scores_mix, labels = scores_la_batch, scores_pa_batch, scores_mix_batch, labels_batch
self.history['valid_la'].append(compute_eer(labels, scores_la))
self.history['valid_pa'].append(compute_eer(labels, scores_pa))
self.history['valid_mix'].append(
compute_eer(labels, scores_mix))
if self.logger:
self.logger.add_scalar('Valid/Valid EER LA',
self.history['valid_la'][-1],
self.total_iters - 1)
self.logger.add_scalar('Valid/Best valid EER LA',
np.min(self.history['valid_la']),
self.total_iters - 1)
self.logger.add_pr_curve('Valid. ROC LA',
labels=labels,
predictions=scores_la,
global_step=self.total_iters - 1)
self.logger.add_histogram('Valid/Scores_LA',
values=scores_la,
global_step=self.total_iters - 1)
self.logger.add_scalar('Valid/Valid EER PA',
self.history['valid_pa'][-1],
self.total_iters - 1)
self.logger.add_scalar('Valid/Best valid EER PA',
np.min(self.history['valid_pa']),
self.total_iters - 1)
self.logger.add_pr_curve('Valid. ROC PA',
labels=labels,
predictions=scores_pa,
global_step=self.total_iters - 1)
self.logger.add_histogram('Valid/Scores_PA',
values=scores_pa,
global_step=self.total_iters - 1)
self.logger.add_scalar('Valid/Valid EER MIX',
self.history['valid_mix'][-1],
self.total_iters - 1)
self.logger.add_scalar('Valid/Best valid EER MIX',
np.min(self.history['valid_mix']),
self.total_iters - 1)
self.logger.add_pr_curve('Valid. ROC MIX',
labels=labels,
predictions=scores_mix,
global_step=self.total_iters - 1)
self.logger.add_histogram('Valid/Scores_MIX',
values=scores_mix,
global_step=self.total_iters - 1)
self.logger.add_histogram('Valid/Labels',
values=labels,
global_step=self.total_iters - 1)
print(
'LA: Current validation loss, best validation loss, and epoch: {:0.4f}, {:0.4f}, {}'
.format(self.history['valid_la'][-1],
np.min(self.history['valid_la']),
1 + np.argmin(self.history['valid_la'])))
print(
'PA: Current validation loss, best validation loss, and epoch: {:0.4f}, {:0.4f}, {}'
.format(self.history['valid_pa'][-1],
np.min(self.history['valid_pa']),
1 + np.argmin(self.history['valid_pa'])))
print(
'MIX: Current validation loss, best validation loss, and epoch: {:0.4f}, {:0.4f}, {}'
.format(self.history['valid_mix'][-1],
np.min(self.history['valid_mix']),
1 + np.argmin(self.history['valid_mix'])))
print('Current LRs (LA, PA, Mixture): {}, {}, {}'.format(
self.optimizer_la.optimizer.param_groups[0]['lr'],
self.optimizer_pa.optimizer.param_groups[0]['lr'],
self.optimizer_mix.optimizer.param_groups[0]['lr']))
self.cur_epoch += 1
if self.cur_epoch % save_every == 0 or self.history['valid_la'][
-1] < np.min([np.inf] + self.history['valid_la'][:-1]
) or self.history['valid_pa'][-1] < np.min(
[np.inf] + self.history['valid_pa'][:-1]
) or self.history['valid_mix'][-1] < np.min(
[np.inf] +
self.history['valid_mix'][:-1]):
self.checkpointing()
print('Training done!')
if self.valid_loader is not None:
print(
'\nLA: Best validation loss and corresponding epoch: {:0.4f}, {}'
.format(np.min(self.history['valid_la']),
1 + np.argmin(self.history['valid_la'])))
print(
'PA: Best validation loss and corresponding epoch: {:0.4f}, {}'
.format(np.min(self.history['valid_pa']),
1 + np.argmin(self.history['valid_pa'])))
print(
'MIX: Best validation loss and corresponding epoch: {:0.4f}, {}'
.format(np.min(self.history['valid_mix']),
1 + np.argmin(self.history['valid_mix'])))
print(
'ALL: Best validation loss and corresponding epoch: {:0.4f}, {}'
.format(np.min(self.history['valid_all']),
1 + np.argmin(self.history['valid_all'])))
def train(self, n_epochs=1, save_every=1):
while (self.cur_epoch < n_epochs):
np.random.seed()
if isinstance(self.train_loader.dataset, Loader):
self.train_loader.dataset.update_lists()
adjust_learning_rate(self.optimizer, self.cur_epoch, self.base_lr, self.patience, self.lr_factor)
if self.verbose>1:
print(' ')
print('Epoch {}/{}'.format(self.cur_epoch+1, n_epochs))
train_iter = tqdm(enumerate(self.train_loader))
else:
train_iter = enumerate(self.train_loader)
if self.pretrain:
ce_epoch=0.0
for t, batch in train_iter:
ce = self.pretrain_step(batch)
self.history['train_loss_batch'].append(ce)
ce_epoch+=ce
self.logger.add_scalar('Train/Cross entropy', ce, self.total_iters)
self.logger.add_scalar('Info/LR', self.optimizer.param_groups[0]['lr'], self.total_iters)
self.total_iters += 1
self.history['train_loss'].append(ce_epoch/(t+1))
if self.verbose>1:
print('Train loss: {:0.4f}'.format(self.history['train_loss'][-1]))
else:
train_loss_epoch=0.0
ce_loss_epoch=0.0
triplet_loss_epoch=0.0
for t, batch in train_iter:
train_loss, ce_loss, triplet_loss = self.train_step(batch)
self.history['train_loss_batch'].append(train_loss)
self.history['ce_loss_batch'].append(ce_loss)
self.history['triplet_loss_batch'].append(triplet_loss)
train_loss_epoch+=train_loss
ce_loss_epoch+=ce_loss
triplet_loss_epoch+=triplet_loss
if self.logger:
self.logger.add_scalar('Train/Total train Loss', train_loss, self.total_iters)
self.logger.add_scalar('Train/Triplet Loss', triplet_loss, self.total_iters)
self.logger.add_scalar('Train/Cross enropy', ce_loss, self.total_iters)
self.logger.add_scalar('Info/LR', self.optimizer.param_groups[0]['lr'], self.total_iters)
self.total_iters += 1
self.history['train_loss'].append(train_loss_epoch/(t+1))
self.history['ce_loss'].append(ce_loss_epoch/(t+1))
self.history['triplet_loss'].append(triplet_loss_epoch/(t+1))
if self.verbose>1:
print(' ')
print('Total train loss: {:0.4f}'.format(self.history['train_loss'][-1]))
print('CE loss: {:0.4f}'.format(self.history['ce_loss'][-1]))
print('triplet_loss loss: {:0.4f}'.format(self.history['triplet_loss'][-1]))
print(' ')
if self.valid_loader is not None:
tot_correct_1, tot_correct_5, tot_ = 0, 0, 0
cos_scores, labels = None, None
for t, batch in enumerate(self.valid_loader):
correct_1, correct_5, total, cos_scores_batch, labels_batch = self.valid(batch)
try:
cos_scores = np.concatenate([cos_scores, cos_scores_batch], 0)
labels = np.concatenate([labels, labels_batch], 0)
except:
cos_scores, labels = cos_scores_batch, labels_batch
tot_correct_1 += correct_1
tot_correct_5 += correct_5
tot_ += total
self.history['cos_eer'].append(compute_eer(labels, cos_scores))
self.history['acc_1'].append(float(tot_correct_1)/tot_)
self.history['acc_5'].append(float(tot_correct_5)/tot_)
if self.logger:
self.logger.add_scalar('Valid/Cosine EER', self.history['cos_eer'][-1], self.total_iters-1)
self.logger.add_scalar('Valid/Best Cosine EER', np.min(self.history['cos_eer']), self.total_iters-1)
self.logger.add_scalar('Valid/ACC-1', self.history['acc_1'][-1], self.total_iters-1)
self.logger.add_scalar('Valid/Best ACC-1', np.max(self.history['acc_1']), self.total_iters-1)
self.logger.add_scalar('Valid/ACC-5', self.history['acc_5'][-1], self.total_iters-1)
self.logger.add_scalar('Valid/Best ACC-5', np.max(self.history['acc_5']), self.total_iters-1)
self.logger.add_pr_curve('Cosine ROC', labels=labels, predictions=cos_scores, global_step=self.total_iters-1)
self.logger.add_histogram('Valid/COS_Scores', values=cos_scores, global_step=self.total_iters-1)
self.logger.add_histogram('Valid/Labels', values=labels, global_step=self.total_iters-1)
if self.verbose>1:
print(' ')
print('Current cos EER, best cos EER, and epoch: {:0.4f}, {:0.4f}, {}'.format(self.history['cos_eer'][-1], np.min(self.history['cos_eer']), 1+np.argmin(self.history['cos_eer'])))
print('Current Top 1 Acc, best Top 1 Acc, and epoch: {:0.4f}, {:0.4f}, {}'.format(self.history['acc_1'][-1], np.max(self.history['acc_1']), 1+np.argmax(self.history['acc_1'])))
print('Current Top 5 Acc, best Top 5 Acc, and epoch: {:0.4f}, {:0.4f}, {}'.format(self.history['acc_5'][-1], np.max(self.history['acc_5']), 1+np.argmax(self.history['acc_5'])))
if self.verbose>1:
print('Current LR: {}'.format(self.optimizer.param_groups[0]['lr']))
self.cur_epoch += 1
if self.valid_loader is not None and self.save_cp and (self.cur_epoch % save_every == 0 or self.history['cos_eer'][-1] < np.min([np.inf]+self.history['cos_eer'][:-1])):
self.checkpointing()
elif self.save_cp and self.cur_epoch % save_every == 0:
self.checkpointing()
if self.verbose>1:
print('Training done!')
if self.valid_loader is not None:
if self.verbose>1:
print('Best cos eer and corresponding epoch: {:0.4f}, {}'.format(np.min(self.history['cos_eer']), 1+np.argmin(self.history['cos_eer'])))
return [np.min(self.history['cos_eer'])]
else:
return [np.min(self.history['train_loss'])]
keep_checkpoint_max=params.keep_checkpoint_max)
estimator = tf.estimator.Estimator(model_fn, params=params, config=config)
if args.mode.lower() == 'train':
""" model:{"Siamese", "SiameseInception", "2ChannelsCNN", "2ChannelsSoftmax" """
tf.logging.info("Starting training model : {} ".format(params.model))
estimator.train(lambda: input_fn(params, is_training=True, repeating=1, is_augment=True))
# estimator.train(lambda: input_fn(params, is_training=True, repeating=1, is_augment=False))
res = estimator.evaluate(lambda: input_fn(params, is_training=False, is_augment=False))
elif args.mode.lower() == 'predict':
res = estimator.predict(lambda: input_fn(params, is_training=False, is_augment=False, only_label=0))
distance_negative = [x['distance'] for x in res]
res = estimator.predict(lambda: input_fn(params, is_training=False, is_augment=False, only_label=1))
distance_positive = [x['distance'] for x in res]
utils.compute_eer(distance_positive=distance_positive, distance_negative=distance_negative)
utils.visualize(distance_positive=distance_positive, distance_negative=distance_negative)
else:
tf.logging.info("Evaluation on test set.")
res = estimator.evaluate(lambda: input_fn(params, is_training=False, is_augment=False))
"""evaluate from first checkpoint to last"""
# checkpoint_file = open(args.model_dir + '/checkpoint', 'r')
# checkpoint_lines = list(checkpoint_file.readlines())
# checkpoint_file.close()
# for i in range(1, len(checkpoint_lines)):
# checkpoint = checkpoint_lines[i].split('\"')[-2]
#
# res = estimator.evaluate(
# lambda: input_fn(params, False, False),
def main(args):
os.environ['CUDA_VISIBLE_DEVICES'] = ','.join(args.gpu_idx)
# if len(args.gpu_idx):
# device = torch.device("cuda" )
# print(device)
# else:
# device = torch.device("cpu")
# dataloader
input_channel = 1
if args.dataset == 'fashionmnist':
args.num_classes = 2
# args.train_path = '/storage/fei/data/'
# args.val_path = '/storage/fei/data/'
# transform = transforms.Compose([transforms.ToTensor(),
# transforms.Normalize((0.1307,), (0.3081,))])
#
# train_set = torchvision.datasets.FashionMNIST(
# root=args.train_path,
# train=True,
# transform=transform
# )
# val_set = torchvision.datasets.FashionMNIST(
# root=args.val_path,
# train=False,
# transform=transform
# )
from keras.datasets import fashion_mnist
(trainX, trainy), (testX, testy) = fashion_mnist.load_data()
# train_set = fashionMNIST(trainX, trainy, real=[5, 7, 9], fake=[0, 1, 2, 3])
# val_set = fashionMNIST(testX, testy, real=[5, 7, 9], fake=[0, 1, 2, 3, 4, 6, 8])
real = [3]
fake_val = [0, 2]
fake_test = [4, 6]
train_set = fashionMNIST(trainX, trainy, real=real, fake=fake_val)
val_set = fashionMNIST(testX, testy, real=real, fake=fake_val)
test_set = fashionMNIST(testX, testy, real=real, fake=fake_test)
else:
raise ValueError(
'Dataset should be: voxceleb1, imagenet, fashionmnist!')
#
train_dataloader = DataLoader(train_set,
batch_size=args.batch_size,
shuffle=True,
num_workers=4,
drop_last=True)
val_dataloader = DataLoader(val_set,
batch_size=args.batch_size,
shuffle=False,
num_workers=4)
test_dataloader = DataLoader(test_set,
batch_size=args.batch_size,
shuffle=False,
num_workers=4)
model = Model_base(args).cuda()
experiment = Experiment(API_KEY, project_name='OC-Softmax')
experiment.log_parameters(vars(args))
experiment.set_name(args.model_dir)
numparams = 0
for f in model.parameters():
if f.requires_grad:
numparams += f.numel()
experiment.log_parameter('Parameters', numparams)
print('Total number of parameters: {}'.format(numparams))
if torch.cuda.device_count() > 1:
print("Let's use", torch.cuda.device_count(), "GPUs!")
model.backbone = nn.DataParallel(model.backbone)
model = model.cuda()
# Optimizer
# optimizer = getattr(optim, args.optim)(model.parameters(), lr=args.lr)
optimizer = torch.optim.SGD(model.parameters(),
args.lr,
momentum=0.9,
weight_decay=1e-4)
# optimizer = optim.SGD([{'params': model.backbone.parameters()},
# {'params': model.softmax_layer.parameters()}],
# lr=args.lr, momentum=0.9, nesterov=False)
# scheduler = StepLR(optimizer, step_size=30, gamma=0.6)
# Save config
model_path = os.path.join(args.exp_dir, args.model_dir)
log_path = os.path.join(model_path, 'logs')
if os.path.exists(log_path):
res = input("Experiment {} already exists, continue? (y/n)".format(
args.model_dir))
print(res)
if res == 'n':
sys.exit()
os.makedirs(log_path, exist_ok=True)
conf_path = os.path.join(log_path, 'conf.yml')
with open(conf_path, 'w') as outfile:
yaml.safe_dump(vars(args), outfile)
log_file = '{}/stats.csv'.format(log_path)
log_content = [
'Epoch', 'tr_acc', 'val_acc', 'test_acc', 'val_eer', 'test_eer',
'tr_loss', 'val_loss', 'test_loss'
]
if not os.path.exists(log_file):
with open(log_file, 'w') as f:
writer = csv.writer(f)
writer.writerow(log_content)
# Train model
tr_step = 0
val_step = 0
new_lr = args.lr
halving = False
best_val_loss = float("-inf")
val_no_impv = 0
# training
iteration = 0
tr_step = 0
for epoch in range(args.epochs):
metric_dic = {}
for m in log_content[1:]:
metric_dic[m] = []
current_lr = adjust_learning_rate(optimizer, tr_step, args.lr)
# print('Epoch:', epoch,'LR:', optimizer.param_groups[0]['lr'], optimizer.param_groups[1]['lr'])
print('Epoch: {}, learning rate: {}'.format(epoch + 1, current_lr))
# train_utils.val_step(spk_classifier, embedding, val_dataloader, iteration, val_log_path)
# Training
model.train()
for data in tqdm(train_dataloader,
desc='{} Training'.format(
args.model_dir)): # mini-batch
# one batch of training data
# input_feature, target = data['input_feature'].to(device), data['target'].to(device)
input_feature, target = data[0].cuda(), data[1].cuda()
# gradient accumulates
optimizer.zero_grad()
# embedding
# embeddings = model.backbone(input_feature)
output, loss = model(input_feature, target)
metric_dic['tr_loss'].append(loss.detach().cpu())
# if args.center > 0:
# l_c = 0
# for i in range(model.embeddings.shape[0]):
# l_c = l_c + 0.5 * (model.embeddings[i] - W[:, target[i]]).pow(2).sum()
# l_c = l_c / model.embeddings.shape[0]
# loss = loss + args.center * l_c
# metric_dic['tr_center_loss'].append(l_c.detch().cpu())
#
# if args.w_ortho > 0:
# W = F.normalize(model.softmax_layer.W, p=2, dim=0)
# l_w_reg = (W.T @ W - torch.eye(W.shape[1]).cuda()).norm(p=2)
# loss = loss + args.w_ortho * l_w_reg
# metric_dic['tr_w_reg'].append(l_w_reg.detach().cpu())
train_acc = utils.accuracy(output, target)[0] # Top-1 acc
metric_dic['tr_acc'].append(train_acc.cpu())
loss.backward()
# torch.nn.utils.clip_grad_norm_(embedding.parameters(), 1.0)
# torch.nn.utils.clip_grad_norm_(spk_classifier.parameters(), 1.0)
optimizer.step()
if iteration % 100 == 0:
print('Train loss: {:.2f}, Acc: {:.2f}%'.format(
loss.item(), train_acc))
iteration += 1
tr_step += 1
# res_dic['tr_loss']['acc'] += l.tolist()
# Validation
if val_dataloader is not None:
model.eval()
outputs = []
targets = []
with torch.no_grad():
for data in tqdm(val_dataloader,
desc='Validation'): # mini-batch
# input_feature, target = data['input_feature'].to(device), data['target'].to(device)
input_feature, target = data[0].cuda(), data[1].cuda()
output, loss = model(input_feature, target)
# val_acc = utils.accuracy(output, target)[0] # Top-1 acc
# metric_dic['val_acc'].append(val_acc.cpu())
metric_dic['val_loss'].append(loss.cpu())
outputs.append(output)
targets.append(target)
metric_dic['val_acc'] = utils.accuracy(
torch.cat(outputs).cpu(),
torch.cat(targets).cpu())[0]
metric_dic['val_acc'] = metric_dic['val_acc'].item()
eer1, _ = utils.compute_eer(
torch.cat(outputs).cpu()[:, 0],
torch.cat(targets).cpu())
eer2, _ = utils.compute_eer(-torch.cat(outputs).cpu()[:, 0],
torch.cat(targets).cpu())
metric_dic['val_eer'] = min(eer1, eer2)
# Test
if test_dataloader is not None:
model.eval()
outputs = []
targets = []
with torch.no_grad():
for data in tqdm(test_dataloader,
desc='Validation'): # mini-batch
# input_feature, target = data['input_feature'].to(device), data['target'].to(device)
input_feature, target = data[0].cuda(), data[1].cuda()
output, loss = model(input_feature, target)
# val_acc = utils.accuracy(output, target)[0] # Top-1 acc
# metric_dic['val_acc'].append(val_acc.cpu())
metric_dic['test_loss'].append(loss.cpu())
outputs.append(output)
targets.append(target)
metric_dic['test_acc'] = utils.accuracy(
torch.cat(outputs).cpu(),
torch.cat(targets).cpu())[0]
metric_dic['test_acc'] = metric_dic['test_acc'].item()
eer1, _ = utils.compute_eer(
torch.cat(outputs).cpu()[:, 0],
torch.cat(targets).cpu())
eer2, _ = utils.compute_eer(-torch.cat(outputs).cpu()[:, 0],
torch.cat(targets).cpu())
metric_dic['test_eer'] = min(eer1, eer2)
for metric in metric_dic.keys():
if isinstance(metric_dic[metric], list):
metric_dic[metric] = np.mean(metric_dic[metric])
if metric[:3] == 'tr_':
with experiment.train():
experiment.log_metric(metric[3:],
metric_dic[metric],
step=tr_step)
if metric[:4] == 'val_':
with experiment.validate():
experiment.log_metric(metric[4:],
metric_dic[metric],
step=tr_step)
pprint(metric_dic)
# Write logs
with open(log_file, 'a') as f:
writer = csv.writer(f)
write_content = [tr_step
] + [metric_dic[m] for m in metric_dic.keys()]
writer.writerow(write_content)
Model_base.save_if_best(save_dir=model_path,
model=model,
optimizer=optimizer,
epoch=tr_step,
tr_metric=metric_dic['tr_acc'],
val_metric=metric_dic['val_eer'],
metric_name='eer',
save_every=10)
