def train(self, n_epochs=1, save_every=1):
while (self.cur_epoch < n_epochs):
np.random.seed()
self.train_loader.dataset.update_lists()
if self.verbose > 0:
print(' ')
print('Epoch {}/{}'.format(self.cur_epoch + 1, n_epochs))
print('Number of training examples given new list: {}'.format(
len(self.train_loader.dataset)))
train_iter = tqdm(enumerate(self.train_loader),
total=len(self.train_loader))
else:
train_iter = enumerate(self.train_loader)
train_loss_epoch = 0.0
if self.softmax and not self.pretrain:
ce_epoch = 0.0
for t, batch in train_iter:
train_loss, ce = self.train_step(batch)
self.history['train_loss_batch'].append(train_loss)
self.history['softmax_batch'].append(ce)
train_loss_epoch += train_loss
ce_epoch += ce
if self.logger:
self.logger.add_scalar('Train/Train Loss', train_loss,
self.total_iters)
self.logger.add_scalar('Train/Triplet Loss',
train_loss - ce,
self.total_iters)
self.logger.add_scalar('Train/Cross enropy', ce,
self.total_iters)
self.logger.add_scalar(
'Info/LR',
self.optimizer.optimizer.param_groups[0]['lr'],
self.total_iters)
self.total_iters += 1
self.history['train_loss'].append(train_loss_epoch / (t + 1))
self.history['softmax'].append(ce_epoch / (t + 1))
if self.verbose > 0:
print(
'Total train loss, Triplet loss, and Cross-entropy: {:0.4f}, {:0.4f}, {:0.4f}'
.format(self.history['train_loss'][-1],
(self.history['train_loss'][-1] -
self.history['softmax'][-1]),
self.history['softmax'][-1]))
elif self.pretrain:
ce_epoch = 0.0
for t, batch in train_iter:
ce = self.pretrain_step(batch)
self.history['train_loss_batch'].append(ce)
ce_epoch += ce
if self.logger:
self.logger.add_scalar('Cross enropy', ce,
self.total_iters)
self.logger.add_scalar(
'Info/LR',
self.optimizer.optimizer.param_groups[0]['lr'],
self.total_iters)
self.total_iters += 1
self.history['train_loss'].append(ce_epoch / (t + 1))
if self.verbose > 0:
print('Train loss: {:0.4f}'.format(
self.history['train_loss'][-1]))
else:
for t, batch in train_iter:
train_loss = self.train_step(batch)
self.history['train_loss_batch'].append(train_loss)
train_loss_epoch += train_loss
if self.logger:
self.logger.add_scalar('Train/Train Loss', train_loss,
self.total_iters)
self.logger.add_scalar(
'Info/LR',
self.optimizer.optimizer.param_groups[0]['lr'],
self.total_iters)
self.total_iters += 1
self.history['train_loss'].append(train_loss_epoch / (t + 1))
if self.verbose > 0:
print('Total train loss, {:0.4f}'.format(
self.history['train_loss'][-1]))
if self.valid_loader is not None:
scores, labels, emb, y_ = None, None, None, None
for t, batch in enumerate(self.valid_loader):
scores_batch, labels_batch, emb_batch, y_batch = self.valid(
batch)
try:
scores = np.concatenate([scores, scores_batch], 0)
labels = np.concatenate([labels, labels_batch], 0)
emb = np.concatenate([emb, emb_batch], 0)
y_ = np.concatenate([y_, y_batch], 0)
except:
scores, labels, emb, y_ = scores_batch, labels_batch, emb_batch, y_batch
self.history['valid_loss'].append(compute_eer(labels, scores))
if self.verbose > 0:
print(
'Current validation loss, best validation loss, and epoch: {:0.4f}, {:0.4f}, {}'
.format(self.history['valid_loss'][-1],
np.min(self.history['valid_loss']),
1 + np.argmin(self.history['valid_loss'])))
if self.logger:
self.logger.add_scalar('Valid/EER',
self.history['valid_loss'][-1],
self.total_iters - 1)
self.logger.add_scalar('Valid/Best EER',
np.min(self.history['valid_loss']),
self.total_iters - 1)
self.logger.add_pr_curve('Valid. ROC',
labels=labels,
predictions=scores,
global_step=self.total_iters - 1)
if emb.shape[0] > 20000:
idxs = np.random.choice(np.arange(emb.shape[0]),
size=20000,
replace=False)
emb, y_ = emb[idxs, :], y_[idxs]
self.logger.add_histogram('Valid/Embeddings',
values=emb,
global_step=self.total_iters - 1)
self.logger.add_histogram('Valid/Scores',
values=scores,
global_step=self.total_iters - 1)
self.logger.add_histogram('Valid/Labels',
values=labels,
global_step=self.total_iters - 1)
if self.verbose > 1:
self.logger.add_embedding(
mat=emb,
metadata=list(y_),
global_step=self.total_iters - 1)
if self.verbose > 0:
print('Current LR: {}'.format(
self.optimizer.optimizer.param_groups[0]['lr']))
self.cur_epoch += 1
if self.valid_loader is not None and self.save_cp and (
self.cur_epoch % save_every == 0
or self.history['valid_loss'][-1] <
np.min([np.inf] + self.history['valid_loss'][:-1])):
self.checkpointing()
elif self.save_cp and self.cur_epoch % save_every == 0:
self.checkpointing()
if self.verbose > 0:
print('Training done!')
if self.valid_loader is not None:
if self.verbose > 0:
print(
'Best validation loss and corresponding epoch: {:0.4f}, {}'
.format(np.min(self.history['valid_loss']),
1 + np.argmin(self.history['valid_loss'])))
return np.min(self.history['valid_loss'])
else:
return np.min(self.history['train_loss'])
def train(self, n_epochs=1, save_every=1):
while (self.cur_epoch < n_epochs):
np.random.seed()
self.train_loader.dataset.update_lists()
if self.verbose>1:
print(' ')
print('Epoch {}/{}'.format(self.cur_epoch+1, n_epochs))
print('Number of training examples given new list: {}'.format(len(self.train_loader.dataset)))
train_iter = tqdm(enumerate(self.train_loader), total=len(self.train_loader))
else:
train_iter = enumerate(self.train_loader)
if self.pretrain:
ce_epoch=0.0
for t, batch in train_iter:
ce = self.pretrain_step(batch)
self.history['train_loss_batch'].append(ce)
ce_epoch+=ce
if self.logger:
self.logger.add_scalar('Train/Cross entropy', ce, self.total_iters)
self.logger.add_scalar('Info/LR', self.optimizer.optimizer.param_groups[0]['lr'], self.total_iters)
self.total_iters += 1
self.history['train_loss'].append(ce_epoch/(t+1))
if self.verbose>1:
print('Train loss: {:0.4f}'.format(self.history['train_loss'][-1]))
else:
train_loss_epoch=0.0
ce_loss_epoch=0.0
bin_loss_epoch=0.0
for t, batch in train_iter:
train_loss, ce_loss, bin_loss = self.train_step(batch)
self.history['train_loss_batch'].append(train_loss)
self.history['ce_loss_batch'].append(ce_loss)
self.history['bin_loss_batch'].append(bin_loss)
train_loss_epoch+=train_loss
ce_loss_epoch+=ce_loss
bin_loss_epoch+=bin_loss
if self.logger:
self.logger.add_scalar('Train/Total train Loss', train_loss, self.total_iters)
self.logger.add_scalar('Train/Binary class. Loss', bin_loss, self.total_iters)
self.logger.add_scalar('Train/Cross enropy', ce_loss, self.total_iters)
self.logger.add_scalar('Info/LR', self.optimizer.optimizer.param_groups[0]['lr'], self.total_iters)
self.total_iters += 1
self.history['train_loss'].append(train_loss_epoch/(t+1))
self.history['ce_loss'].append(ce_loss_epoch/(t+1))
self.history['bin_loss'].append(bin_loss_epoch/(t+1))
if self.verbose>1:
print(' ')
print('Total train loss: {:0.4f}'.format(self.history['train_loss'][-1]))
print('CE loss: {:0.4f}'.format(self.history['ce_loss'][-1]))
print('Binary classification loss: {:0.4f}'.format(self.history['bin_loss'][-1]))
print(' ')
if self.valid_loader is not None:
e2e_scores, cos_scores, labels, emb, y_ = None, None, None, None, None
for t, batch in enumerate(self.valid_loader):
e2e_scores_batch, cos_scores_batch, labels_batch, emb_batch, y_batch = self.valid(batch)
try:
e2e_scores = np.concatenate([e2e_scores, e2e_scores_batch], 0)
cos_scores = np.concatenate([cos_scores, cos_scores_batch], 0)
labels = np.concatenate([labels, labels_batch], 0)
emb = np.concatenate([emb, emb_batch], 0)
y_ = np.concatenate([y_, y_batch], 0)
except:
e2e_scores, cos_scores, labels, emb, y_ = e2e_scores_batch, cos_scores_batch, labels_batch, emb_batch, y_batch
fus_scores = (e2e_scores + 0.5*(cos_scores+1.))*0.5
self.history['e2e_eer'].append(compute_eer(labels, e2e_scores))
self.history['cos_eer'].append(compute_eer(labels, cos_scores))
self.history['fus_eer'].append(compute_eer(labels, fus_scores))
if self.logger:
self.logger.add_scalar('Valid/E2E EER', self.history['e2e_eer'][-1], self.total_iters-1)
self.logger.add_scalar('Valid/Best E2E EER', np.min(self.history['e2e_eer']), self.total_iters-1)
self.logger.add_scalar('Valid/Cosine EER', self.history['cos_eer'][-1], self.total_iters-1)
self.logger.add_scalar('Valid/Best Cosine EER', np.min(self.history['cos_eer']), self.total_iters-1)
self.logger.add_scalar('Valid/Fus EER', self.history['fus_eer'][-1], self.total_iters-1)
self.logger.add_scalar('Valid/Best Fus EER', np.min(self.history['fus_eer']), self.total_iters-1)
self.logger.add_pr_curve('E2E ROC', labels=labels, predictions=e2e_scores, global_step=self.total_iters-1)
self.logger.add_pr_curve('Cosine ROC', labels=labels, predictions=cos_scores, global_step=self.total_iters-1)
self.logger.add_pr_curve('Fus ROC', labels=labels, predictions=fus_scores, global_step=self.total_iters-1)
if emb.shape[0]>20000:
idxs = np.random.choice(np.arange(emb.shape[0]), size=20000, replace=False)
emb, y_ = emb[idxs, :], y_[idxs]
self.logger.add_histogram('Valid/Embeddings', values=emb, global_step=self.total_iters-1)
self.logger.add_histogram('Valid/COS_Scores', values=cos_scores, global_step=self.total_iters-1)
self.logger.add_histogram('Valid/E2E_Scores', values=e2e_scores, global_step=self.total_iters-1)
self.logger.add_histogram('Valid/FUS_Scores', values=fus_scores, global_step=self.total_iters-1)
self.logger.add_histogram('Valid/Labels', values=labels, global_step=self.total_iters-1)
if self.verbose>1:
self.logger.add_embedding(mat=emb, metadata=list(y_), global_step=self.total_iters-1)
if self.verbose>1:
print(' ')
print('Current e2e EER, best e2e EER, and epoch: {:0.4f}, {:0.4f}, {}'.format(self.history['e2e_eer'][-1], np.min(self.history['e2e_eer']), 1+np.argmin(self.history['e2e_eer'])))
print('Current cos EER, best cos EER, and epoch: {:0.4f}, {:0.4f}, {}'.format(self.history['cos_eer'][-1], np.min(self.history['cos_eer']), 1+np.argmin(self.history['cos_eer'])))
print('Current fus EER, best fus EER, and epoch: {:0.4f}, {:0.4f}, {}'.format(self.history['fus_eer'][-1], np.min(self.history['fus_eer']), 1+np.argmin(self.history['fus_eer'])))
if self.verbose>1:
print('Current LR: {}'.format(self.optimizer.optimizer.param_groups[0]['lr']))
self.cur_epoch += 1
if self.valid_loader is not None and self.save_cp and (self.cur_epoch % save_every == 0 or self.history['e2e_eer'][-1] < np.min([np.inf]+self.history['e2e_eer'][:-1]) or self.history['cos_eer'][-1] < np.min([np.inf]+self.history['cos_eer'][:-1])):
self.checkpointing()
elif self.save_cp and self.cur_epoch % save_every == 0:
self.checkpointing()
if self.verbose>1:
print('Training done!')
if self.valid_loader is not None:
if self.verbose>1:
print('Best e2e eer and corresponding epoch: {:0.4f}, {}'.format(np.min(self.history['e2e_eer']), 1+np.argmin(self.history['e2e_eer'])))
print('Best cos eer and corresponding epoch: {:0.4f}, {}'.format(np.min(self.history['cos_eer']), 1+np.argmin(self.history['cos_eer'])))
print('Best fus eer and corresponding epoch: {:0.4f}, {}'.format(np.min(self.history['fus_eer']), 1+np.argmin(self.history['fus_eer'])))
return [np.min(self.history['e2e_eer']), np.min(self.history['cos_eer'])]
else:
return [np.min(self.history['train_loss'])]
def train(self, n_epochs=1, save_every=1):
while (self.cur_epoch < n_epochs):
np.random.seed()
if self.verbose>0:
print(' ')
print('Epoch {}/{}'.format(self.cur_epoch+1, n_epochs))
train_iter = tqdm(enumerate(self.train_loader))
else:
train_iter = enumerate(self.train_loader)
train_loss_epoch=0.0
if self.softmax and not self.pretrain:
ce_epoch=0.0
for t, batch in train_iter:
train_loss, ce = self.train_step(batch)
self.history['train_loss_batch'].append(train_loss)
self.history['softmax_batch'].append(ce)
train_loss_epoch+=train_loss
ce_epoch+=ce
self.total_iters += 1
self.history['train_loss'].append(train_loss_epoch/(t+1))
self.history['softmax'].append(ce_epoch/(t+1))
if self.verbose>0:
print('Total train loss, Triplet loss, and Cross-entropy: {:0.4f}, {:0.4f}, {:0.4f}'.format(self.history['train_loss'][-1], (self.history['train_loss'][-1]-self.history['softmax'][-1]), self.history['softmax'][-1]))
elif self.pretrain:
ce_epoch=0.0
for t, batch in train_iter:
ce = self.pretrain_step(batch)
self.history['train_loss_batch'].append(ce)
ce_epoch+=ce
self.total_iters += 1
self.history['train_loss'].append(ce_epoch/(t+1))
if self.verbose>0:
print('Train loss: {:0.4f}'.format(self.history['train_loss'][-1]))
else:
for t, batch in train_iter:
train_loss = self.train_step(batch)
self.history['train_loss_batch'].append(train_loss)
train_loss_epoch+=train_loss
self.total_iters += 1
self.history['train_loss'].append(train_loss_epoch/(t+1))
if self.verbose>0:
print('Total train loss, {:0.4f}'.format(self.history['train_loss'][-1]))
if self.valid_loader is not None:
scores, labels = None, None
for t, batch in enumerate(self.valid_loader):
scores_batch, labels_batch = self.valid(batch)
try:
scores = np.concatenate([scores, scores_batch], 0)
labels = np.concatenate([labels, labels_batch], 0)
except:
scores, labels = scores_batch, labels_batch
self.history['valid_loss'].append(compute_eer(labels, scores))
if self.verbose>0:
print('Current validation loss, best validation loss, and epoch: {:0.4f}, {:0.4f}, {}'.format(self.history['valid_loss'][-1], np.min(self.history['valid_loss']), 1+np.argmin(self.history['valid_loss'])))
self.scheduler.step(self.history['valid_loss'][-1])
else:
self.scheduler.step()
if self.verbose>0:
print('Current LR: {}'.format(self.optimizer.param_groups[0]['lr']))
self.cur_epoch += 1
if self.valid_loader is not None and self.save_cp and (self.cur_epoch % save_every == 0 or self.history['valid_loss'][-1] < np.min([np.inf]+self.history['valid_loss'][:-1])):
self.checkpointing()
elif self.save_cp and self.cur_epoch % save_every == 0:
self.checkpointing()
if self.verbose>0:
print('Training done!')
if self.valid_loader is not None:
if self.verbose>0:
print('Best validation loss and corresponding epoch: {:0.4f}, {}'.format(np.min(self.history['valid_loss']), 1+np.argmin(self.history['valid_loss'])))
return np.min(self.history['valid_loss'])
else:
return np.min(self.history['train_loss'])
def train(self, n_epochs=1, save_every=1):
while (self.cur_epoch < n_epochs):
np.random.seed()
if self.verbose > 0:
print(' ')
print('Epoch {}/{}'.format(self.cur_epoch + 1, n_epochs))
train_iter = tqdm(enumerate(self.train_loader))
else:
train_iter = enumerate(self.train_loader)
if self.pretrain:
ce_epoch = 0.0
for t, batch in train_iter:
ce = self.pretrain_step(batch)
self.history['train_loss_batch'].append(ce)
ce_epoch += ce
self.total_iters += 1
self.history['train_loss'].append(ce_epoch / (t + 1))
if self.verbose > 0:
print('Train loss: {:0.4f}'.format(
self.history['train_loss'][-1]))
else:
train_loss_epoch = 0.0
ce_loss_epoch = 0.0
bin_loss_epoch = 0.0
for t, batch in train_iter:
train_loss, ce_loss, bin_loss = self.train_step(batch)
self.history['train_loss_batch'].append(train_loss)
self.history['ce_loss_batch'].append(ce_loss)
self.history['bin_loss_batch'].append(bin_loss)
train_loss_epoch += train_loss
ce_loss_epoch += ce_loss
bin_loss_epoch += bin_loss
self.total_iters += 1
self.history['train_loss'].append(train_loss_epoch / (t + 1))
self.history['ce_loss'].append(ce_loss_epoch / (t + 1))
self.history['bin_loss'].append(bin_loss_epoch / (t + 1))
if self.verbose > 0:
print(' ')
print('Total train loss: {:0.4f}'.format(
self.history['train_loss'][-1]))
print('CE loss: {:0.4f}'.format(
self.history['ce_loss'][-1]))
print('Binary classification loss: {:0.4f}'.format(
self.history['bin_loss'][-1]))
print(' ')
if self.valid_loader is not None:
e2e_scores, cos_scores, labels = None, None, None
for t, batch in enumerate(self.valid_loader):
e2e_scores_batch, cos_scores_batch, labels_batch = self.valid(
batch)
try:
e2e_scores = np.concatenate(
[e2e_scores, e2e_scores_batch], 0)
cos_scores = np.concatenate(
[cos_scores, cos_scores_batch], 0)
labels = np.concatenate([labels, labels_batch], 0)
except:
e2e_scores, cos_scores, labels = e2e_scores_batch, cos_scores_batch, labels_batch
self.history['e2e_eer'].append(compute_eer(labels, e2e_scores))
self.history['cos_eer'].append(compute_eer(labels, cos_scores))
if self.verbose > 0:
print(' ')
print(
'Current e2e EER, best e2e EER, and epoch: {:0.4f}, {:0.4f}, {}'
.format(self.history['e2e_eer'][-1],
np.min(self.history['e2e_eer']),
1 + np.argmin(self.history['e2e_eer'])))
print(
'Current cos EER, best cos EER, and epoch: {:0.4f}, {:0.4f}, {}'
.format(self.history['cos_eer'][-1],
np.min(self.history['cos_eer']),
1 + np.argmin(self.history['cos_eer'])))
self.scheduler.step(
np.min([
self.history['e2e_eer'][-1],
self.history['cos_eer'][-1]
]))
else:
self.scheduler.step()
if self.verbose > 0:
print('Current LR: {}'.format(
self.optimizer.param_groups[0]['lr']))
self.cur_epoch += 1
if self.valid_loader is not None and self.save_cp and (
self.cur_epoch % save_every == 0
or self.history['e2e_eer'][-1] <
np.min([np.inf] + self.history['e2e_eer'][:-1])
or self.history['cos_eer'][-1] <
np.min([np.inf] + self.history['cos_eer'][:-1])):
self.checkpointing()
elif self.save_cp and self.cur_epoch % save_every == 0:
self.checkpointing()
if self.verbose > 0:
print('Training done!')
if self.valid_loader is not None:
if self.verbose > 0:
print(
'Best e2e eer and corresponding epoch: {:0.4f}, {}'.format(
np.min(self.history['e2e_eer']),
1 + np.argmin(self.history['e2e_eer'])))
print(
'Best cos eer and corresponding epoch: {:0.4f}, {}'.format(
np.min(self.history['cos_eer']),
1 + np.argmin(self.history['cos_eer'])))
return [
np.min(self.history['e2e_eer']),
np.min(self.history['cos_eer'])
]
else:
return [np.min(self.history['train_loss'])]