def main(config): testIvectorDataset = TestIvectorDataset(config) pSVM = PSVM(testIvectorDataset.ivecs) pSVM.load_model(config.ipath2model) pSVM.scoring(testIvectorDataset, testIvectorDataset.idx2utt, config.opath2score) compute_eer(config.opath2score, config.trials)
def train(self, n_epochs=1, save_every=1): while (self.cur_epoch < n_epochs): print('Epoch {}/{}'.format(self.cur_epoch + 1, n_epochs)) train_iter = tqdm(enumerate(self.train_loader)) train_loss_epoch = 0.0 for t, batch in train_iter: train_loss = self.train_step(batch) self.history['train_loss_batch'].append(train_loss) train_loss_epoch += train_loss self.total_iters += 1 self.history['train_loss'].append(train_loss_epoch / (t + 1)) print('Total train loss: {:0.4f}'.format( self.history['train_loss'][-1])) if self.valid_loader is not None: scores, labels = None, None for t, batch in enumerate(self.valid_loader): scores_batch, labels_batch = self.valid(batch) try: scores = np.concatenate([scores, scores_batch], 0) labels = np.concatenate([labels, labels_batch], 0) except: scores, labels = scores_batch, labels_batch self.history['valid_loss'].append(compute_eer(labels, scores)) print( 'Current validation loss, best validation loss, and epoch: {:0.4f}, {:0.4f}, {}' .format(self.history['valid_loss'][-1], np.min(self.history['valid_loss']), 1 + np.argmin(self.history['valid_loss']))) self.scheduler.step() print('Current LR: {}'.format( self.optimizer.param_groups[0]['lr'])) self.cur_epoch += 1 if self.cur_epoch % save_every == 0 or self.history['valid_loss'][ -1] < np.min([np.inf] + self.history['valid_loss'][:-1]): self.checkpointing() print('Training done!') if self.valid_loader is not None: print('Best validation loss and corresponding epoch: {:0.4f}, {}'. format(np.min(self.history['valid_loss']), 1 + np.argmin(self.history['valid_loss']))) return np.min(self.history['valid_loss'])
def validate_verification(cfg, model, test_loader): batch_time = AverageMeter('Time', ':6.3f') progress = ProgressMeter(len(test_loader), batch_time, prefix='Test: ', logger=logger) # switch to evaluate mode model.eval() labels, distances = [], [] with torch.no_grad(): end = time.time() for i, (input1, input2, label) in enumerate(test_loader): input1 = input1.cuda(non_blocking=True).squeeze(0) input2 = input2.cuda(non_blocking=True).squeeze(0) label = label.cuda(non_blocking=True) # compute output outputs1 = model(input1).mean(dim=0).unsqueeze(0) outputs2 = model(input2).mean(dim=0).unsqueeze(0) dists = F.cosine_similarity(outputs1, outputs2) dists = dists.data.cpu().numpy() distances.append(dists) labels.append(label.data.cpu().numpy()) # measure elapsed time batch_time.update(time.time() - end) end = time.time() if i % 2000 == 0: progress.print(i) labels = np.array([sublabel for label in labels for sublabel in label]) distances = np.array( [subdist for dist in distances for subdist in dist]) eer = compute_eer(distances, labels) logger.info('Test EER: {:.8f}'.format(np.mean(eer))) return eer
def train(self, n_epochs=1, save_every=1): while (self.cur_epoch < n_epochs): np.random.seed() if self.verbose>0: print(' ') print('Epoch {}/{}'.format(self.cur_epoch+1, n_epochs)) train_iter = tqdm(enumerate(self.train_loader)) else: train_iter = enumerate(self.train_loader) if self.pretrain: ce_epoch=0.0 for t, batch in train_iter: ce = self.pretrain_step(batch) self.history['train_loss_batch'].append(ce) ce_epoch+=ce self.total_iters += 1 self.history['train_loss'].append(ce_epoch/(t+1)) if self.verbose>0: print('Train loss: {:0.4f}'.format(self.history['train_loss'][-1])) else: train_loss_epoch=0.0 ce_loss_epoch=0.0 bin_loss_epoch=0.0 for t, batch in train_iter: train_loss, ce_loss, bin_loss = self.train_step(batch) self.history['train_loss_batch'].append(train_loss) self.history['ce_loss_batch'].append(ce_loss) self.history['bin_loss_batch'].append(bin_loss) train_loss_epoch+=train_loss ce_loss_epoch+=ce_loss bin_loss_epoch+=bin_loss self.total_iters += 1 self.history['train_loss'].append(train_loss_epoch/(t+1)) self.history['ce_loss'].append(ce_loss_epoch/(t+1)) self.history['bin_loss'].append(bin_loss_epoch/(t+1)) if self.verbose>0: print(' ') print('Total train loss: {:0.4f}'.format(self.history['train_loss'][-1])) print('CE loss: {:0.4f}'.format(self.history['ce_loss'][-1])) print('Binary classification loss: {:0.4f}'.format(self.history['bin_loss'][-1])) print(' ') if self.valid_loader is not None: tot_correct, tot_ = 0, 0 e2e_scores, cos_scores, labels = None, None, None for t, batch in enumerate(self.valid_loader): correct, total, e2e_scores_batch, cos_scores_batch, labels_batch = self.valid(batch) try: e2e_scores = np.concatenate([e2e_scores, e2e_scores_batch], 0) cos_scores = np.concatenate([cos_scores, cos_scores_batch], 0) labels = np.concatenate([labels, labels_batch], 0) except: e2e_scores, cos_scores, labels = e2e_scores_batch, cos_scores_batch, labels_batch tot_correct += correct tot_ += total self.history['e2e_eer'].append(compute_eer(labels, e2e_scores)) self.history['cos_eer'].append(compute_eer(labels, cos_scores)) self.history['ErrorRate'].append(1.-float(tot_correct)/tot_) if self.verbose>0: print(' ') print('Current e2e EER, best e2e EER, and epoch: {:0.4f}, {:0.4f}, {}'.format(self.history['e2e_eer'][-1], np.min(self.history['e2e_eer']), 1+np.argmin(self.history['e2e_eer']))) print('Current cos EER, best cos EER, and epoch: {:0.4f}, {:0.4f}, {}'.format(self.history['cos_eer'][-1], np.min(self.history['cos_eer']), 1+np.argmin(self.history['cos_eer']))) print('Current Error rate, best Error rate, and epoch: {:0.4f}, {:0.4f}, {}'.format(self.history['ErrorRate'][-1], np.min(self.history['ErrorRate']), 1+np.argmin(self.history['ErrorRate']))) self.scheduler.step(np.min([self.history['e2e_eer'][-1], self.history['cos_eer'][-1]])) else: self.scheduler.step() if self.verbose>0: print('Current LR: {}'.format(self.optimizer.param_groups[0]['lr'])) self.cur_epoch += 1 if self.valid_loader is not None and self.save_cp and (self.cur_epoch % save_every == 0 or self.history['e2e_eer'][-1] < np.min([np.inf]+self.history['e2e_eer'][:-1]) or self.history['cos_eer'][-1] < np.min([np.inf]+self.history['cos_eer'][:-1])): self.checkpointing() elif self.save_cp and self.cur_epoch % save_every == 0: self.checkpointing() if self.verbose>0: print('Training done!') if self.valid_loader is not None: if self.verbose>0: print('Best e2e eer and corresponding epoch: {:0.4f}, {}'.format(np.min(self.history['e2e_eer']), 1+np.argmin(self.history['e2e_eer']))) print('Best cos eer and corresponding epoch: {:0.4f}, {}'.format(np.min(self.history['cos_eer']), 1+np.argmin(self.history['cos_eer']))) return [np.min(self.history['e2e_eer']), np.min(self.history['cos_eer']), np.min(self.history['ErrorRate'])] else: return [np.min(self.history['train_loss'])]
def train(self, n_epochs=1, save_every=1): while self.cur_epoch < n_epochs: np.random.seed() if self.verbose > 0: print(' ') print('Epoch {}/{}'.format(self.cur_epoch + 1, n_epochs)) train_iter = tqdm(enumerate(self.train_loader)) else: train_iter = enumerate(self.train_loader) ce = 0.0 triplet_loss = 0.0 train_loss = 0.0 # Train step for t, batch in train_iter: ce_batch, triplet_loss_batch = self.train_step(batch) ce += ce_batch triplet_loss += triplet_loss_batch train_loss += ce_batch + triplet_loss_batch self.history['train_loss_batch'].append(ce_batch + triplet_loss_batch) self.history['triplet_loss_batch'].append(triplet_loss_batch) self.history['ce_loss_batch'].append(ce_batch) self.total_iters += 1 self.history['train_loss'].append(train_loss / (t + 1)) self.history['triplet_loss'].append(triplet_loss / (t + 1)) self.history['ce_loss'].append(ce / (t + 1)) if self.verbose > 0: print(' ') print( 'Total train loss, Triplet loss, and Cross-entropy: {:0.4f}, {:0.4f}, {:0.4f}' .format(self.history['train_loss'][-1], self.history['triplet_loss'][-1], self.history['ce_loss'][-1])) # Validation tot_correct = 0 tot_ = 0 scores, labels = None, None for t, batch in enumerate(self.valid_loader): correct, total, scores_batch, labels_batch = self.valid(batch) try: scores = np.concatenate([scores, scores_batch], 0) labels = np.concatenate([labels, labels_batch], 0) except: scores, labels = scores_batch, labels_batch tot_correct += correct tot_ += total self.history['EER'].append(compute_eer(labels, scores)) self.history['ErrorRate'].append(1. - float(tot_correct) / tot_) if self.verbose > 0: print(' ') print( 'Current, best validation error rate, and epoch: {:0.4f}, {:0.4f}, {}' .format(self.history['ErrorRate'][-1], np.min(self.history['ErrorRate']), 1 + np.argmin(self.history['ErrorRate']))) print(' ') print( 'Current, best validation EER, and epoch: {:0.4f}, {:0.4f}, {}' .format(self.history['EER'][-1], np.min(self.history['EER']), 1 + np.argmin(self.history['EER']))) self.scheduler.step(self.history['ErrorRate'][-1]) if self.verbose > 0: print(' ') print('Current LR: {}'.format( self.optimizer.param_groups[0]['lr'])) if self.save_cp and ( self.cur_epoch % save_every == 0 or (self.history['ErrorRate'][-1] < np.min([np.inf] + self.history['ErrorRate'][:-1])) or (self.history['EER'][-1] < np.min([np.inf] + self.history['EER'][:-1]))): self.checkpointing() self.cur_epoch += 1 if self.verbose > 0: print('Training done!') if self.valid_loader is not None: print('Best error rate and corresponding epoch: {:0.4f}, {}'. format(np.min(self.history['ErrorRate']), 1 + np.argmin(self.history['ErrorRate']))) print('Best EER and corresponding epoch: {:0.4f}, {}'.format( np.min(self.history['EER']), 1 + np.argmin(self.history['EER']))) return np.min(self.history['ErrorRate'])
def train(self, n_epochs=1, save_every=1): while self.cur_epoch < n_epochs: np.random.seed() if isinstance(self.train_loader.dataset, Loader): self.train_loader.dataset.update_lists() adjust_learning_rate(self.optimizer, self.cur_epoch, self.base_lr) if self.verbose > 0: print(' ') print('Epoch {}/{}'.format(self.cur_epoch + 1, n_epochs)) train_iter = tqdm(enumerate(self.train_loader)) else: train_iter = enumerate(self.train_loader) ce = 0.0 triplet_loss = 0.0 train_loss = 0.0 # Train step for t, batch in train_iter: ce_batch, triplet_loss_batch = self.train_step(batch) ce += ce_batch triplet_loss += triplet_loss_batch train_loss += ce_batch + triplet_loss_batch self.history['train_loss_batch'].append(ce_batch + triplet_loss_batch) self.history['triplet_loss_batch'].append(triplet_loss_batch) self.history['ce_loss_batch'].append(ce_batch) self.total_iters += 1 self.history['train_loss'].append(train_loss / (t + 1)) self.history['triplet_loss'].append(triplet_loss / (t + 1)) self.history['ce_loss'].append(ce / (t + 1)) if self.verbose > 0: print(' ') print( 'Total train loss, Triplet loss, and Cross-entropy: {:0.4f}, {:0.4f}, {:0.4f}' .format(self.history['train_loss'][-1], self.history['triplet_loss'][-1], self.history['ce_loss'][-1])) # Validation tot_correct_1, tot_correct_5, tot_ = 0, 0, 0 scores, labels = None, None for t, batch in enumerate(self.valid_loader): correct_1, correct_5, total, scores_batch, labels_batch = self.valid( batch) try: scores = np.concatenate([scores, scores_batch], 0) labels = np.concatenate([labels, labels_batch], 0) except: scores, labels = scores_batch, labels_batch tot_correct_1 += correct_1 tot_correct_5 += correct_5 tot_ += total self.history['EER'].append(compute_eer(labels, scores)) self.history['acc_1'].append(float(tot_correct_1) / tot_) self.history['acc_5'].append(float(tot_correct_5) / tot_) if self.verbose > 0: print(' ') print( 'Current, best validation EER, and epoch: {:0.4f}, {:0.4f}, {}' .format(self.history['EER'][-1], np.min(self.history['EER']), 1 + np.argmin(self.history['EER']))) print( 'Current Top 1 Acc, best Top 1 Acc, and epoch: {:0.4f}, {:0.4f}, {}' .format(self.history['acc_1'][-1], np.max(self.history['acc_1']), 1 + np.argmax(self.history['acc_1']))) print( 'Current Top 5 Acc, best Top 5 Acc, and epoch: {:0.4f}, {:0.4f}, {}' .format(self.history['acc_5'][-1], np.max(self.history['acc_5']), 1 + np.argmax(self.history['acc_5']))) if self.verbose > 0: print(' ') print('Current LR: {}'.format( self.optimizer.param_groups[0]['lr'])) if self.save_cp and (self.cur_epoch % save_every == 0 or self.history['EER'][-1] < np.min([np.inf] + self.history['EER'][:-1])): self.checkpointing() self.cur_epoch += 1 if self.verbose > 0: print('Training done!') if self.valid_loader is not None: print('Best EER and corresponding epoch: {:0.4f}, {}'.format( np.min(self.history['EER']), 1 + np.argmin(self.history['EER']))) return np.min(self.history['EER'])
def train(self, n_epochs=1, save_every=1): while (self.cur_epoch < n_epochs): np.random.seed() if self.verbose > 0: print(' ') print('Epoch {}/{}'.format(self.cur_epoch + 1, n_epochs)) train_iter = tqdm(enumerate(self.train_loader)) else: train_iter = enumerate(self.train_loader) ce_asv_epoch = 0.0 ce_lid_epoch = 0.0 for t, batch in train_iter: ce_asv, ce_lid = self.train_step(batch) self.history['ce_asv_batch'].append(ce_asv) self.history['ce_lid_batch'].append(ce_lid) ce_asv_epoch += ce_asv ce_lid_epoch += ce_lid self.total_iters += 1 self.history['ce_asv'].append(ce_asv_epoch / (t + 1)) self.history['ce_lid'].append(ce_lid_epoch / (t + 1)) if self.verbose > 0: print( 'Total train loss, ASV CE, and LID CE: {:0.4f}, {:0.4f}, {:0.4f}' .format( self.history['ce_asv'][-1] + self.history['ce_lid'][-1], self.history['ce_asv'][-1], self.history['ce_lid'][-1])) scores, labels = None, None for t, batch in enumerate(self.valid_loader): scores_batch, labels_batch = self.valid(batch) try: scores = np.concatenate([scores, scores_batch], 0) labels = np.concatenate([labels, labels_batch], 0) except: scores, labels = scores_batch, labels_batch self.history['valid_loss'].append(compute_eer(labels, scores)) if self.verbose > 0: print( 'Current validation loss, best validation loss, and epoch: {:0.4f}, {:0.4f}, {}' .format(self.history['valid_loss'][-1], np.min(self.history['valid_loss']), 1 + np.argmin(self.history['valid_loss']))) self.scheduler_s.step(self.history['valid_loss'][-1]) self.scheduler_l.step(self.history['valid_loss'][-1]) if self.verbose > 0: print('Current LR: {}'.format( self.optimizer.param_groups[0]['lr'])) self.cur_epoch += 1 if self.save_cp and ( self.cur_epoch % save_every == 0 or self.history['valid_loss'][-1] < np.min([np.inf] + self.history['valid_loss'][:-1])): self.checkpointing() if self.verbose > 0: print('Training done!') if self.verbose > 0: print('Best validation loss and corresponding epoch: {:0.4f}, {}'. format(np.min(self.history['valid_loss']), 1 + np.argmin(self.history['valid_loss']))) return np.min(self.history['valid_loss'])
def evaluate(self): if self.verbose > 0: print('\nIteration - Epoch: {} - {}'.format( self.total_iters, self.cur_epoch)) e2e_scores, cos_scores, labels = None, None, None for t, batch in enumerate(self.valid_loader): e2e_scores_batch, cos_scores_batch, labels_batch = self.valid( batch) try: e2e_scores = np.concatenate([e2e_scores, e2e_scores_batch], 0) cos_scores = np.concatenate([cos_scores, cos_scores_batch], 0) labels = np.concatenate([labels, labels_batch], 0) except: e2e_scores, cos_scores, labels = e2e_scores_batch, cos_scores_batch, labels_batch self.history['e2e_eer'].append(compute_eer(labels, e2e_scores)) self.history['cos_eer'].append(compute_eer(labels, cos_scores)) if self.history['e2e_eer'][-1] < self.best_e2e_eer: self.best_e2e_eer = self.history['e2e_eer'][-1] self.best_e2e_eer_epoch = self.cur_epoch self.best_e2e_eer_iteration = self.total_iters if self.history['cos_eer'][-1] < self.best_cos_eer: self.best_cos_eer = self.history['cos_eer'][-1] self.best_cos_eer_epoch = self.cur_epoch self.best_cos_eer_iteration = self.total_iters if self.logger: self.logger.add_scalar('Valid/E2E EER', self.history['e2e_eer'][-1], self.total_iters) self.logger.add_scalar('Valid/Best E2E EER', np.min(self.history['e2e_eer']), self.total_iters) self.logger.add_scalar('Valid/Cosine EER', self.history['cos_eer'][-1], self.total_iters) self.logger.add_scalar('Valid/Best Cosine EER', np.min(self.history['cos_eer']), self.total_iters) self.logger.add_pr_curve('E2E ROC', labels=labels, predictions=e2e_scores, global_step=self.total_iters) self.logger.add_pr_curve('Cosine ROC', labels=labels, predictions=cos_scores, global_step=self.total_iters) self.logger.add_histogram('Valid/COS_Scores', values=cos_scores, global_step=self.total_iters) self.logger.add_histogram('Valid/E2E_Scores', values=e2e_scores, global_step=self.total_iters) self.logger.add_histogram('Valid/Labels', values=labels, global_step=self.total_iters) if self.verbose > 0: print(' ') print( 'Current e2e EER, best e2e EER, and epoch - iteration: {:0.4f}, {:0.4f}, {}, {}' .format(self.history['e2e_eer'][-1], np.min(self.history['e2e_eer']), self.best_e2e_eer_epoch, self.best_e2e_eer_iteration)) print( 'Current cos EER, best cos EER, and epoch - iteration: {:0.4f}, {:0.4f}, {}, {}' .format(self.history['cos_eer'][-1], np.min(self.history['cos_eer']), self.best_cos_eer_epoch, self.best_cos_eer_iteration))
def train(self, n_epochs=1, save_every=1): while (self.cur_epoch < n_epochs): np.random.seed() if isinstance(self.train_loader.dataset, Loader): self.train_loader.dataset.update_lists() if self.verbose > 0: print(' ') print('Epoch {}/{}'.format(self.cur_epoch + 1, n_epochs)) train_iter = tqdm(enumerate(self.train_loader)) else: train_iter = enumerate(self.train_loader) if self.pretrain: ce_epoch = 0.0 for t, batch in train_iter: ce = self.pretrain_step(batch) self.history['train_loss_batch'].append(ce) ce_epoch += ce if self.logger: self.logger.add_scalar('Train/Cross entropy', ce, self.total_iters) self.logger.add_scalar( 'Info/LR', self.optimizer.param_groups[0]['lr'], self.total_iters) self.total_iters += 1 self.history['train_loss'].append(ce_epoch / (t + 1)) if self.verbose > 0: print('Train loss: {:0.4f}'.format( self.history['train_loss'][-1])) else: train_loss_epoch = 0.0 ce_loss_epoch = 0.0 bin_loss_epoch = 0.0 for t, batch in train_iter: train_loss, ce_loss, bin_loss = self.train_step(batch) self.history['train_loss_batch'].append(train_loss) self.history['ce_loss_batch'].append(ce_loss) self.history['bin_loss_batch'].append(bin_loss) train_loss_epoch += train_loss ce_loss_epoch += ce_loss bin_loss_epoch += bin_loss if self.logger: self.logger.add_scalar('Train/Total train Loss', train_loss, self.total_iters) self.logger.add_scalar('Train/Binary class. Loss', bin_loss, self.total_iters) self.logger.add_scalar('Train/Cross enropy', ce_loss, self.total_iters) self.logger.add_scalar( 'Info/LR', self.optimizer.param_groups[0]['lr'], self.total_iters) self.total_iters += 1 self.history['train_loss'].append(train_loss_epoch / (t + 1)) self.history['ce_loss'].append(ce_loss_epoch / (t + 1)) self.history['bin_loss'].append(bin_loss_epoch / (t + 1)) if self.verbose > 0: print(' ') print('Total train loss: {:0.4f}'.format( self.history['train_loss'][-1])) print('CE loss: {:0.4f}'.format( self.history['ce_loss'][-1])) print('Binary classification loss: {:0.4f}'.format( self.history['bin_loss'][-1])) print(' ') if self.valid_loader is not None: e2e_scores, cos_scores, labels = None, None, None for t, batch in enumerate(self.valid_loader): e2e_scores_batch, cos_scores_batch, labels_batch = self.valid( batch) try: e2e_scores = np.concatenate( [e2e_scores, e2e_scores_batch], 0) cos_scores = np.concatenate( [cos_scores, cos_scores_batch], 0) labels = np.concatenate([labels, labels_batch], 0) except: e2e_scores, cos_scores, labels = e2e_scores_batch, cos_scores_batch, labels_batch self.history['e2e_eer'].append(compute_eer(labels, e2e_scores)) self.history['cos_eer'].append(compute_eer(labels, cos_scores)) if self.logger: self.logger.add_scalar('Valid/E2E EER', self.history['e2e_eer'][-1], self.total_iters - 1) self.logger.add_scalar('Valid/Best E2E EER', np.min(self.history['e2e_eer']), self.total_iters - 1) self.logger.add_scalar('Valid/Cosine EER', self.history['cos_eer'][-1], self.total_iters - 1) self.logger.add_scalar('Valid/Best Cosine EER', np.min(self.history['cos_eer']), self.total_iters - 1) self.logger.add_pr_curve('E2E ROC', labels=labels, predictions=e2e_scores, global_step=self.total_iters - 1) self.logger.add_pr_curve('Cosine ROC', labels=labels, predictions=cos_scores, global_step=self.total_iters - 1) self.logger.add_histogram('Valid/COS_Scores', values=cos_scores, global_step=self.total_iters - 1) self.logger.add_histogram('Valid/E2E_Scores', values=e2e_scores, global_step=self.total_iters - 1) self.logger.add_histogram('Valid/Labels', values=labels, global_step=self.total_iters - 1) if self.verbose > 0: print(' ') print( 'Current e2e EER, best e2e EER, and epoch: {:0.4f}, {:0.4f}, {}' .format(self.history['e2e_eer'][-1], np.min(self.history['e2e_eer']), 1 + np.argmin(self.history['e2e_eer']))) print( 'Current cos EER, best cos EER, and epoch: {:0.4f}, {:0.4f}, {}' .format(self.history['cos_eer'][-1], np.min(self.history['cos_eer']), 1 + np.argmin(self.history['cos_eer']))) self.scheduler.step( np.min([ self.history['e2e_eer'][-1], self.history['cos_eer'][-1] ])) else: self.scheduler.step() if self.verbose > 0: print('Current LR: {}'.format( self.optimizer.param_groups[0]['lr'])) self.cur_epoch += 1 if self.valid_loader is not None and self.save_cp and ( self.cur_epoch % save_every == 0 or self.history['e2e_eer'][-1] < np.min([np.inf] + self.history['e2e_eer'][:-1]) or self.history['cos_eer'][-1] < np.min([np.inf] + self.history['cos_eer'][:-1])): self.checkpointing() elif self.save_cp and self.cur_epoch % save_every == 0: self.checkpointing() if self.verbose > 0: print('Training done!') if self.valid_loader is not None: if self.verbose > 0: print( 'Best e2e eer and corresponding epoch: {:0.4f}, {}'.format( np.min(self.history['e2e_eer']), 1 + np.argmin(self.history['e2e_eer']))) print( 'Best cos eer and corresponding epoch: {:0.4f}, {}'.format( np.min(self.history['cos_eer']), 1 + np.argmin(self.history['cos_eer']))) return [ np.min(self.history['e2e_eer']), np.min(self.history['cos_eer']) ] else: return [np.min(self.history['train_loss'])]
def train(self, n_epochs=1, save_every=1): while (self.cur_epoch < n_epochs): if self.verbose > 0: print(' ') print('Epoch {}/{}'.format(self.cur_epoch + 1, n_epochs)) train_iter = tqdm(enumerate(self.train_loader), total=len(self.train_loader)) else: train_iter = enumerate(self.train_loader) train_loss_epoch = 0.0 for t, batch in train_iter: train_loss = self.train_step(batch) self.history['train_loss_batch'].append(train_loss) train_loss_epoch += train_loss if self.logger: self.logger.add_scalar('Train/Train Loss', train_loss, self.total_iters) self.logger.add_scalar( 'Info/LR', self.optimizer.optimizer.param_groups[0]['lr'], self.total_iters) self.logger.add_scalar('Info/Epoch', self.cur_epoch, self.total_iters) self.total_iters += 1 self.history['train_loss'].append(train_loss_epoch / (t + 1)) if self.verbose > 0: print('Total train loss: {:0.4f}'.format( self.history['train_loss'][-1])) if self.valid_loader is not None: scores, labels = None, None for t, batch in enumerate(self.valid_loader): scores_batch, labels_batch = self.valid(batch) try: scores = np.concatenate([scores, scores_batch], 0) labels = np.concatenate([labels, labels_batch], 0) except: scores, labels = scores_batch, labels_batch self.history['valid_loss'].append(compute_eer(labels, scores)) if self.logger: self.logger.add_scalar('Valid/Valid EER', self.history['valid_loss'][-1], self.total_iters) self.logger.add_scalar('Valid/Best valid EER', np.min(self.history['valid_loss']), self.total_iters) self.logger.add_pr_curve('E2E ROC', labels=labels, predictions=scores, global_step=self.total_iters) self.logger.add_histogram('Valid/Scores', values=scores, global_step=self.total_iters - 1) self.logger.add_histogram('Valid/Labels', values=labels, global_step=self.total_iters - 1) if self.verbose > 0: print( 'Current validation loss, best validation loss, and epoch: {:0.4f}, {:0.4f}, {}' .format(self.history['valid_loss'][-1], np.min(self.history['valid_loss']), 1 + np.argmin(self.history['valid_loss']))) if self.verbose > 0: print('Current LR: {}'.format( self.optimizer.optimizer.param_groups[0]['lr'])) self.cur_epoch += 1 if (self.cur_epoch % save_every == 0 or self.history['valid_loss'][-1] < np.min([np.inf] + self.history['valid_loss'][:-1]) ) and self.save_cp: self.checkpointing() if self.verbose > 0: print('Training done!') if self.valid_loader is not None: if self.verbose > 0: print( 'Best validation loss and corresponding epoch: {:0.4f}, {}' .format(np.min(self.history['valid_loss']), 1 + np.argmin(self.history['valid_loss']))) return np.min(self.history['valid_loss'])
def train(self, n_epochs=1, save_every=1): while (self.cur_epoch < n_epochs): print('Epoch {}/{}'.format(self.cur_epoch + 1, n_epochs)) train_iter = tqdm(enumerate(self.train_loader)) train_loss_epoch = 0.0 train_all_epoch = 0.0 train_la_epoch = 0.0 train_pa_epoch = 0.0 train_mix_epoch = 0.0 for t, batch in train_iter: train_loss, train_all, train_la, train_pa, train_mix = self.train_step( batch) self.history['train_loss_batch'].append(train_loss) self.history['train_all_batch'].append(train_all) self.history['train_la_batch'].append(train_la) self.history['train_pa_batch'].append(train_pa) self.history['train_mix_batch'].append(train_mix) train_loss_epoch += train_loss train_all_epoch += train_all train_la_epoch += train_la train_pa_epoch += train_pa train_mix_epoch += train_mix if self.logger: self.logger.add_scalar('Train/Total train loss', train_loss, self.total_iters) self.logger.add_scalar('Train/Train Loss mixture', train_all, self.total_iters) self.logger.add_scalar('Train/Train Loss LA', train_la, self.total_iters) self.logger.add_scalar('Train/Train Loss PA', train_pa, self.total_iters) self.logger.add_scalar('Train/Train Loss MIX', train_mix, self.total_iters) self.logger.add_scalar( 'Info/LR_LA', self.optimizer_la.optimizer.param_groups[0]['lr'], self.total_iters) self.logger.add_scalar( 'Info/LR_PA', self.optimizer_pa.optimizer.param_groups[0]['lr'], self.total_iters) self.logger.add_scalar( 'Info/LR_MIX', self.optimizer_mix.optimizer.param_groups[0]['lr'], self.total_iters) self.total_iters += 1 self.history['train_loss'].append(train_loss_epoch / (t + 1)) self.history['train_all'].append(train_all_epoch / (t + 1)) self.history['train_la'].append(train_la_epoch / (t + 1)) self.history['train_pa'].append(train_pa_epoch / (t + 1)) self.history['train_mix'].append(train_mix_epoch / (t + 1)) print( 'Total train loss, loss_all, loss_la, loss_pa, loss_mix: {:0.4f}, {:0.4f}, {:0.4f}, {:0.4f}, {:0.4f}' .format(self.history['train_loss'][-1], self.history['train_all'][-1], self.history['train_la'][-1], self.history['train_pa'][-1], self.history['train_mix'][-1])) scores_all, scores_la, scores_pa, scores_mix, labels = None, None, None, None, None if self.train_mode == 'mix': for t, batch in enumerate(self.valid_loader): scores_all_batch, scores_la_batch, scores_pa_batch, scores_mix_batch, labels_batch = self.valid( batch) try: scores_all = np.concatenate( [scores_all, scores_all_batch], 0) scores_la = np.concatenate( [scores_la, scores_la_batch], 0) scores_pa = np.concatenate( [scores_pa, scores_pa_batch], 0) scores_mix = np.concatenate( [scores_mix, scores_mix_batch], 0) labels = np.concatenate([labels, labels_batch], 0) except: scores_all, scores_la, scores_pa, scores_mix, labels = scores_all_batch, scores_la_batch, scores_pa_batch, scores_mix_batch, labels_batch self.history['valid_all'].append( compute_eer(labels, scores_all)) self.history['valid_la'].append(compute_eer(labels, scores_la)) self.history['valid_pa'].append(compute_eer(labels, scores_pa)) self.history['valid_mix'].append( compute_eer(labels, scores_mix)) if self.logger: self.logger.add_scalar('Valid/Valid EER mixture', self.history['valid_all'][-1], self.total_iters - 1) self.logger.add_scalar('Valid/Best valid EER mixture', np.min(self.history['valid_all']), self.total_iters - 1) self.logger.add_pr_curve('Valid. ROC mixture', labels=labels, predictions=scores_all, global_step=self.total_iters - 1) self.logger.add_histogram('Valid/Scores_Mixture', values=scores_all, global_step=self.total_iters - 1) print( 'ALL: Current validation loss, best validation loss, and epoch: {:0.4f}, {:0.4f}, {}' .format(self.history['valid_all'][-1], np.min(self.history['valid_all']), 1 + np.argmin(self.history['valid_all']))) else: for t, batch in enumerate(self.valid_loader): scores_la_batch, scores_pa_batch, scores_mix_batch, labels_batch = self.valid( batch) try: scores_la = np.concatenate( [scores_la, scores_la_batch], 0) scores_pa = np.concatenate( [scores_pa, scores_pa_batch], 0) scores_mix = np.concatenate( [scores_mix, scores_mix_batch], 0) labels = np.concatenate([labels, labels_batch], 0) except: scores_la, scores_pa, scores_mix, labels = scores_la_batch, scores_pa_batch, scores_mix_batch, labels_batch self.history['valid_la'].append(compute_eer(labels, scores_la)) self.history['valid_pa'].append(compute_eer(labels, scores_pa)) self.history['valid_mix'].append( compute_eer(labels, scores_mix)) if self.logger: self.logger.add_scalar('Valid/Valid EER LA', self.history['valid_la'][-1], self.total_iters - 1) self.logger.add_scalar('Valid/Best valid EER LA', np.min(self.history['valid_la']), self.total_iters - 1) self.logger.add_pr_curve('Valid. ROC LA', labels=labels, predictions=scores_la, global_step=self.total_iters - 1) self.logger.add_histogram('Valid/Scores_LA', values=scores_la, global_step=self.total_iters - 1) self.logger.add_scalar('Valid/Valid EER PA', self.history['valid_pa'][-1], self.total_iters - 1) self.logger.add_scalar('Valid/Best valid EER PA', np.min(self.history['valid_pa']), self.total_iters - 1) self.logger.add_pr_curve('Valid. ROC PA', labels=labels, predictions=scores_pa, global_step=self.total_iters - 1) self.logger.add_histogram('Valid/Scores_PA', values=scores_pa, global_step=self.total_iters - 1) self.logger.add_scalar('Valid/Valid EER MIX', self.history['valid_mix'][-1], self.total_iters - 1) self.logger.add_scalar('Valid/Best valid EER MIX', np.min(self.history['valid_mix']), self.total_iters - 1) self.logger.add_pr_curve('Valid. ROC MIX', labels=labels, predictions=scores_mix, global_step=self.total_iters - 1) self.logger.add_histogram('Valid/Scores_MIX', values=scores_mix, global_step=self.total_iters - 1) self.logger.add_histogram('Valid/Labels', values=labels, global_step=self.total_iters - 1) print( 'LA: Current validation loss, best validation loss, and epoch: {:0.4f}, {:0.4f}, {}' .format(self.history['valid_la'][-1], np.min(self.history['valid_la']), 1 + np.argmin(self.history['valid_la']))) print( 'PA: Current validation loss, best validation loss, and epoch: {:0.4f}, {:0.4f}, {}' .format(self.history['valid_pa'][-1], np.min(self.history['valid_pa']), 1 + np.argmin(self.history['valid_pa']))) print( 'MIX: Current validation loss, best validation loss, and epoch: {:0.4f}, {:0.4f}, {}' .format(self.history['valid_mix'][-1], np.min(self.history['valid_mix']), 1 + np.argmin(self.history['valid_mix']))) print('Current LRs (LA, PA, Mixture): {}, {}, {}'.format( self.optimizer_la.optimizer.param_groups[0]['lr'], self.optimizer_pa.optimizer.param_groups[0]['lr'], self.optimizer_mix.optimizer.param_groups[0]['lr'])) self.cur_epoch += 1 if self.cur_epoch % save_every == 0 or self.history['valid_la'][ -1] < np.min([np.inf] + self.history['valid_la'][:-1] ) or self.history['valid_pa'][-1] < np.min( [np.inf] + self.history['valid_pa'][:-1] ) or self.history['valid_mix'][-1] < np.min( [np.inf] + self.history['valid_mix'][:-1]): self.checkpointing() print('Training done!') if self.valid_loader is not None: print( '\nLA: Best validation loss and corresponding epoch: {:0.4f}, {}' .format(np.min(self.history['valid_la']), 1 + np.argmin(self.history['valid_la']))) print( 'PA: Best validation loss and corresponding epoch: {:0.4f}, {}' .format(np.min(self.history['valid_pa']), 1 + np.argmin(self.history['valid_pa']))) print( 'MIX: Best validation loss and corresponding epoch: {:0.4f}, {}' .format(np.min(self.history['valid_mix']), 1 + np.argmin(self.history['valid_mix']))) print( 'ALL: Best validation loss and corresponding epoch: {:0.4f}, {}' .format(np.min(self.history['valid_all']), 1 + np.argmin(self.history['valid_all'])))
def train(self, n_epochs=1, save_every=1): while (self.cur_epoch < n_epochs): np.random.seed() if isinstance(self.train_loader.dataset, Loader): self.train_loader.dataset.update_lists() adjust_learning_rate(self.optimizer, self.cur_epoch, self.base_lr, self.patience, self.lr_factor) if self.verbose>1: print(' ') print('Epoch {}/{}'.format(self.cur_epoch+1, n_epochs)) train_iter = tqdm(enumerate(self.train_loader)) else: train_iter = enumerate(self.train_loader) if self.pretrain: ce_epoch=0.0 for t, batch in train_iter: ce = self.pretrain_step(batch) self.history['train_loss_batch'].append(ce) ce_epoch+=ce self.logger.add_scalar('Train/Cross entropy', ce, self.total_iters) self.logger.add_scalar('Info/LR', self.optimizer.param_groups[0]['lr'], self.total_iters) self.total_iters += 1 self.history['train_loss'].append(ce_epoch/(t+1)) if self.verbose>1: print('Train loss: {:0.4f}'.format(self.history['train_loss'][-1])) else: train_loss_epoch=0.0 ce_loss_epoch=0.0 triplet_loss_epoch=0.0 for t, batch in train_iter: train_loss, ce_loss, triplet_loss = self.train_step(batch) self.history['train_loss_batch'].append(train_loss) self.history['ce_loss_batch'].append(ce_loss) self.history['triplet_loss_batch'].append(triplet_loss) train_loss_epoch+=train_loss ce_loss_epoch+=ce_loss triplet_loss_epoch+=triplet_loss if self.logger: self.logger.add_scalar('Train/Total train Loss', train_loss, self.total_iters) self.logger.add_scalar('Train/Triplet Loss', triplet_loss, self.total_iters) self.logger.add_scalar('Train/Cross enropy', ce_loss, self.total_iters) self.logger.add_scalar('Info/LR', self.optimizer.param_groups[0]['lr'], self.total_iters) self.total_iters += 1 self.history['train_loss'].append(train_loss_epoch/(t+1)) self.history['ce_loss'].append(ce_loss_epoch/(t+1)) self.history['triplet_loss'].append(triplet_loss_epoch/(t+1)) if self.verbose>1: print(' ') print('Total train loss: {:0.4f}'.format(self.history['train_loss'][-1])) print('CE loss: {:0.4f}'.format(self.history['ce_loss'][-1])) print('triplet_loss loss: {:0.4f}'.format(self.history['triplet_loss'][-1])) print(' ') if self.valid_loader is not None: tot_correct_1, tot_correct_5, tot_ = 0, 0, 0 cos_scores, labels = None, None for t, batch in enumerate(self.valid_loader): correct_1, correct_5, total, cos_scores_batch, labels_batch = self.valid(batch) try: cos_scores = np.concatenate([cos_scores, cos_scores_batch], 0) labels = np.concatenate([labels, labels_batch], 0) except: cos_scores, labels = cos_scores_batch, labels_batch tot_correct_1 += correct_1 tot_correct_5 += correct_5 tot_ += total self.history['cos_eer'].append(compute_eer(labels, cos_scores)) self.history['acc_1'].append(float(tot_correct_1)/tot_) self.history['acc_5'].append(float(tot_correct_5)/tot_) if self.logger: self.logger.add_scalar('Valid/Cosine EER', self.history['cos_eer'][-1], self.total_iters-1) self.logger.add_scalar('Valid/Best Cosine EER', np.min(self.history['cos_eer']), self.total_iters-1) self.logger.add_scalar('Valid/ACC-1', self.history['acc_1'][-1], self.total_iters-1) self.logger.add_scalar('Valid/Best ACC-1', np.max(self.history['acc_1']), self.total_iters-1) self.logger.add_scalar('Valid/ACC-5', self.history['acc_5'][-1], self.total_iters-1) self.logger.add_scalar('Valid/Best ACC-5', np.max(self.history['acc_5']), self.total_iters-1) self.logger.add_pr_curve('Cosine ROC', labels=labels, predictions=cos_scores, global_step=self.total_iters-1) self.logger.add_histogram('Valid/COS_Scores', values=cos_scores, global_step=self.total_iters-1) self.logger.add_histogram('Valid/Labels', values=labels, global_step=self.total_iters-1) if self.verbose>1: print(' ') print('Current cos EER, best cos EER, and epoch: {:0.4f}, {:0.4f}, {}'.format(self.history['cos_eer'][-1], np.min(self.history['cos_eer']), 1+np.argmin(self.history['cos_eer']))) print('Current Top 1 Acc, best Top 1 Acc, and epoch: {:0.4f}, {:0.4f}, {}'.format(self.history['acc_1'][-1], np.max(self.history['acc_1']), 1+np.argmax(self.history['acc_1']))) print('Current Top 5 Acc, best Top 5 Acc, and epoch: {:0.4f}, {:0.4f}, {}'.format(self.history['acc_5'][-1], np.max(self.history['acc_5']), 1+np.argmax(self.history['acc_5']))) if self.verbose>1: print('Current LR: {}'.format(self.optimizer.param_groups[0]['lr'])) self.cur_epoch += 1 if self.valid_loader is not None and self.save_cp and (self.cur_epoch % save_every == 0 or self.history['cos_eer'][-1] < np.min([np.inf]+self.history['cos_eer'][:-1])): self.checkpointing() elif self.save_cp and self.cur_epoch % save_every == 0: self.checkpointing() if self.verbose>1: print('Training done!') if self.valid_loader is not None: if self.verbose>1: print('Best cos eer and corresponding epoch: {:0.4f}, {}'.format(np.min(self.history['cos_eer']), 1+np.argmin(self.history['cos_eer']))) return [np.min(self.history['cos_eer'])] else: return [np.min(self.history['train_loss'])]
keep_checkpoint_max=params.keep_checkpoint_max) estimator = tf.estimator.Estimator(model_fn, params=params, config=config) if args.mode.lower() == 'train': """ model:{"Siamese", "SiameseInception", "2ChannelsCNN", "2ChannelsSoftmax" """ tf.logging.info("Starting training model : {} ".format(params.model)) estimator.train(lambda: input_fn(params, is_training=True, repeating=1, is_augment=True)) # estimator.train(lambda: input_fn(params, is_training=True, repeating=1, is_augment=False)) res = estimator.evaluate(lambda: input_fn(params, is_training=False, is_augment=False)) elif args.mode.lower() == 'predict': res = estimator.predict(lambda: input_fn(params, is_training=False, is_augment=False, only_label=0)) distance_negative = [x['distance'] for x in res] res = estimator.predict(lambda: input_fn(params, is_training=False, is_augment=False, only_label=1)) distance_positive = [x['distance'] for x in res] utils.compute_eer(distance_positive=distance_positive, distance_negative=distance_negative) utils.visualize(distance_positive=distance_positive, distance_negative=distance_negative) else: tf.logging.info("Evaluation on test set.") res = estimator.evaluate(lambda: input_fn(params, is_training=False, is_augment=False)) """evaluate from first checkpoint to last""" # checkpoint_file = open(args.model_dir + '/checkpoint', 'r') # checkpoint_lines = list(checkpoint_file.readlines()) # checkpoint_file.close() # for i in range(1, len(checkpoint_lines)): # checkpoint = checkpoint_lines[i].split('\"')[-2] # # res = estimator.evaluate( # lambda: input_fn(params, False, False),
def main(args): os.environ['CUDA_VISIBLE_DEVICES'] = ','.join(args.gpu_idx) # if len(args.gpu_idx): # device = torch.device("cuda" ) # print(device) # else: # device = torch.device("cpu") # dataloader input_channel = 1 if args.dataset == 'fashionmnist': args.num_classes = 2 # args.train_path = '/storage/fei/data/' # args.val_path = '/storage/fei/data/' # transform = transforms.Compose([transforms.ToTensor(), # transforms.Normalize((0.1307,), (0.3081,))]) # # train_set = torchvision.datasets.FashionMNIST( # root=args.train_path, # train=True, # transform=transform # ) # val_set = torchvision.datasets.FashionMNIST( # root=args.val_path, # train=False, # transform=transform # ) from keras.datasets import fashion_mnist (trainX, trainy), (testX, testy) = fashion_mnist.load_data() # train_set = fashionMNIST(trainX, trainy, real=[5, 7, 9], fake=[0, 1, 2, 3]) # val_set = fashionMNIST(testX, testy, real=[5, 7, 9], fake=[0, 1, 2, 3, 4, 6, 8]) real = [3] fake_val = [0, 2] fake_test = [4, 6] train_set = fashionMNIST(trainX, trainy, real=real, fake=fake_val) val_set = fashionMNIST(testX, testy, real=real, fake=fake_val) test_set = fashionMNIST(testX, testy, real=real, fake=fake_test) else: raise ValueError( 'Dataset should be: voxceleb1, imagenet, fashionmnist!') # train_dataloader = DataLoader(train_set, batch_size=args.batch_size, shuffle=True, num_workers=4, drop_last=True) val_dataloader = DataLoader(val_set, batch_size=args.batch_size, shuffle=False, num_workers=4) test_dataloader = DataLoader(test_set, batch_size=args.batch_size, shuffle=False, num_workers=4) model = Model_base(args).cuda() experiment = Experiment(API_KEY, project_name='OC-Softmax') experiment.log_parameters(vars(args)) experiment.set_name(args.model_dir) numparams = 0 for f in model.parameters(): if f.requires_grad: numparams += f.numel() experiment.log_parameter('Parameters', numparams) print('Total number of parameters: {}'.format(numparams)) if torch.cuda.device_count() > 1: print("Let's use", torch.cuda.device_count(), "GPUs!") model.backbone = nn.DataParallel(model.backbone) model = model.cuda() # Optimizer # optimizer = getattr(optim, args.optim)(model.parameters(), lr=args.lr) optimizer = torch.optim.SGD(model.parameters(), args.lr, momentum=0.9, weight_decay=1e-4) # optimizer = optim.SGD([{'params': model.backbone.parameters()}, # {'params': model.softmax_layer.parameters()}], # lr=args.lr, momentum=0.9, nesterov=False) # scheduler = StepLR(optimizer, step_size=30, gamma=0.6) # Save config model_path = os.path.join(args.exp_dir, args.model_dir) log_path = os.path.join(model_path, 'logs') if os.path.exists(log_path): res = input("Experiment {} already exists, continue? (y/n)".format( args.model_dir)) print(res) if res == 'n': sys.exit() os.makedirs(log_path, exist_ok=True) conf_path = os.path.join(log_path, 'conf.yml') with open(conf_path, 'w') as outfile: yaml.safe_dump(vars(args), outfile) log_file = '{}/stats.csv'.format(log_path) log_content = [ 'Epoch', 'tr_acc', 'val_acc', 'test_acc', 'val_eer', 'test_eer', 'tr_loss', 'val_loss', 'test_loss' ] if not os.path.exists(log_file): with open(log_file, 'w') as f: writer = csv.writer(f) writer.writerow(log_content) # Train model tr_step = 0 val_step = 0 new_lr = args.lr halving = False best_val_loss = float("-inf") val_no_impv = 0 # training iteration = 0 tr_step = 0 for epoch in range(args.epochs): metric_dic = {} for m in log_content[1:]: metric_dic[m] = [] current_lr = adjust_learning_rate(optimizer, tr_step, args.lr) # print('Epoch:', epoch,'LR:', optimizer.param_groups[0]['lr'], optimizer.param_groups[1]['lr']) print('Epoch: {}, learning rate: {}'.format(epoch + 1, current_lr)) # train_utils.val_step(spk_classifier, embedding, val_dataloader, iteration, val_log_path) # Training model.train() for data in tqdm(train_dataloader, desc='{} Training'.format( args.model_dir)): # mini-batch # one batch of training data # input_feature, target = data['input_feature'].to(device), data['target'].to(device) input_feature, target = data[0].cuda(), data[1].cuda() # gradient accumulates optimizer.zero_grad() # embedding # embeddings = model.backbone(input_feature) output, loss = model(input_feature, target) metric_dic['tr_loss'].append(loss.detach().cpu()) # if args.center > 0: # l_c = 0 # for i in range(model.embeddings.shape[0]): # l_c = l_c + 0.5 * (model.embeddings[i] - W[:, target[i]]).pow(2).sum() # l_c = l_c / model.embeddings.shape[0] # loss = loss + args.center * l_c # metric_dic['tr_center_loss'].append(l_c.detch().cpu()) # # if args.w_ortho > 0: # W = F.normalize(model.softmax_layer.W, p=2, dim=0) # l_w_reg = (W.T @ W - torch.eye(W.shape[1]).cuda()).norm(p=2) # loss = loss + args.w_ortho * l_w_reg # metric_dic['tr_w_reg'].append(l_w_reg.detach().cpu()) train_acc = utils.accuracy(output, target)[0] # Top-1 acc metric_dic['tr_acc'].append(train_acc.cpu()) loss.backward() # torch.nn.utils.clip_grad_norm_(embedding.parameters(), 1.0) # torch.nn.utils.clip_grad_norm_(spk_classifier.parameters(), 1.0) optimizer.step() if iteration % 100 == 0: print('Train loss: {:.2f}, Acc: {:.2f}%'.format( loss.item(), train_acc)) iteration += 1 tr_step += 1 # res_dic['tr_loss']['acc'] += l.tolist() # Validation if val_dataloader is not None: model.eval() outputs = [] targets = [] with torch.no_grad(): for data in tqdm(val_dataloader, desc='Validation'): # mini-batch # input_feature, target = data['input_feature'].to(device), data['target'].to(device) input_feature, target = data[0].cuda(), data[1].cuda() output, loss = model(input_feature, target) # val_acc = utils.accuracy(output, target)[0] # Top-1 acc # metric_dic['val_acc'].append(val_acc.cpu()) metric_dic['val_loss'].append(loss.cpu()) outputs.append(output) targets.append(target) metric_dic['val_acc'] = utils.accuracy( torch.cat(outputs).cpu(), torch.cat(targets).cpu())[0] metric_dic['val_acc'] = metric_dic['val_acc'].item() eer1, _ = utils.compute_eer( torch.cat(outputs).cpu()[:, 0], torch.cat(targets).cpu()) eer2, _ = utils.compute_eer(-torch.cat(outputs).cpu()[:, 0], torch.cat(targets).cpu()) metric_dic['val_eer'] = min(eer1, eer2) # Test if test_dataloader is not None: model.eval() outputs = [] targets = [] with torch.no_grad(): for data in tqdm(test_dataloader, desc='Validation'): # mini-batch # input_feature, target = data['input_feature'].to(device), data['target'].to(device) input_feature, target = data[0].cuda(), data[1].cuda() output, loss = model(input_feature, target) # val_acc = utils.accuracy(output, target)[0] # Top-1 acc # metric_dic['val_acc'].append(val_acc.cpu()) metric_dic['test_loss'].append(loss.cpu()) outputs.append(output) targets.append(target) metric_dic['test_acc'] = utils.accuracy( torch.cat(outputs).cpu(), torch.cat(targets).cpu())[0] metric_dic['test_acc'] = metric_dic['test_acc'].item() eer1, _ = utils.compute_eer( torch.cat(outputs).cpu()[:, 0], torch.cat(targets).cpu()) eer2, _ = utils.compute_eer(-torch.cat(outputs).cpu()[:, 0], torch.cat(targets).cpu()) metric_dic['test_eer'] = min(eer1, eer2) for metric in metric_dic.keys(): if isinstance(metric_dic[metric], list): metric_dic[metric] = np.mean(metric_dic[metric]) if metric[:3] == 'tr_': with experiment.train(): experiment.log_metric(metric[3:], metric_dic[metric], step=tr_step) if metric[:4] == 'val_': with experiment.validate(): experiment.log_metric(metric[4:], metric_dic[metric], step=tr_step) pprint(metric_dic) # Write logs with open(log_file, 'a') as f: writer = csv.writer(f) write_content = [tr_step ] + [metric_dic[m] for m in metric_dic.keys()] writer.writerow(write_content) Model_base.save_if_best(save_dir=model_path, model=model, optimizer=optimizer, epoch=tr_step, tr_metric=metric_dic['tr_acc'], val_metric=metric_dic['val_eer'], metric_name='eer', save_every=10)