def test(self, dataset, set): test_WER = 0 test_loss = 0 num_examples = 0 self.model.eval() for idx, batch in enumerate(tqdm(dataset.loader)): #enumerate(dataset.loader): x,y,T,U,_ = batch batch_size = len(x) num_examples += batch_size log_probs = self.model(x,y,T,U) loss = -log_probs.mean() test_loss += loss.item() * batch_size WERs = [] guesses = self.model.infer(x[:2], T[:2]) print("decoding subset") for i in range(2): #range(batch_size): guess = guesses[i][:U[i]] truth = y[i].cpu().data.numpy().tolist()[:U[i]] WERs.append(compute_WER(dataset.phoneme_tokenizer.DecodeIds(truth), dataset.phoneme_tokenizer.DecodeIds(guess))) WER = np.array(WERs).mean() test_WER += WER * batch_size print("guess:", dataset.phoneme_tokenizer.DecodeIds(guess)) print("truth:", dataset.phoneme_tokenizer.DecodeIds(truth)) print("WER: ", compute_WER(dataset.phoneme_tokenizer.DecodeIds(truth), dataset.phoneme_tokenizer.DecodeIds(guess))) print("") test_loss /= num_examples self.scheduler.step() test_WER /= num_examples results = {"loss" : test_loss, "WER" : test_WER, "set": set} self.log(results) return test_WER, test_loss
def test(self, dataset, set, log_results=True, print_results=True): test_WER = 0 test_loss = 0 num_examples = 0 self.model.eval() FLOPs = [] for idx, batch in enumerate(dataset.loader): x, y, T, U, _ = batch batch_size = len(x) num_examples += batch_size log_probs, _, _ = self.model(x, y, T, U, self.config.deterministic_test) loss = -log_probs.mean() test_loss += loss.item() * batch_size WERs = [] guesses, p_big, I_big = self.model.infer( x, self.config.deterministic_test, T) batch_FLOPs = (1 - I_big.mean( )) * self.model.FLOPs_small + I_big.mean() * self.model.FLOPs_big FLOPs += [batch_FLOPs.item()] * batch_size for i in range(batch_size): guess = guesses[i][:U[i]] truth = y[i].cpu().data.numpy().tolist()[:U[i]] guess_decoded = self.remove_repeated_silence( dataset.tokenizer.DecodeIds(guess)) truth_decoded = self.remove_repeated_silence( dataset.tokenizer.DecodeIds(truth)) WERs.append(compute_WER(truth_decoded, guess_decoded)) WER = np.array(WERs).mean() test_WER += WER * batch_size if print_results: print("guess:", guess_decoded) print("truth:", truth_decoded) print("WER: ", compute_WER(truth_decoded, guess_decoded)) print("p_big: ", p_big.mean().item()) print("") test_loss /= num_examples self.scheduler.step() test_WER /= num_examples FLOPs = np.array(FLOPs) FLOPs_mean = FLOPs.mean() FLOPs_std = FLOPs.std() results = { "loss": test_loss, "WER": test_WER, "FLOPs_mean": FLOPs_mean, "FLOPs_std": FLOPs_std, "set": set, "surprisal-triggered": self.model.sample_based_on_surprisal_during_testing } if log_results: self.log(results) return test_WER, test_loss, FLOPs_mean, FLOPs_std
def train(self, dataset, print_interval=100): train_WER = 0 train_loss = 0 num_examples = 0 self.model.train() for g in self.optimizer.param_groups: print("Current learning rate:", g['lr']) #self.model.print_frozen() for idx, batch in enumerate(tqdm(dataset.loader)): x,y,T,U,idxs = batch batch_size = len(x) log_probs = self.model(x,y,T,U) loss = -log_probs.mean() if torch.isnan(loss): print("nan detected!") sys.exit() self.optimizer.zero_grad() loss.backward() clip_value = 5; torch.nn.utils.clip_grad_norm_(self.model.parameters(), clip_value) self.optimizer.step() train_loss += loss.item() * batch_size num_examples += batch_size if idx % print_interval == 0: print("loss: " + str(loss.cpu().data.numpy().item())) guess = self.model.infer(x[:1], T[:1])[0][:U[0]] print("guess:", dataset.phoneme_tokenizer.DecodeIds(guess)) truth = y[0].cpu().data.numpy().tolist()[:U[0]] print("truth:", dataset.phoneme_tokenizer.DecodeIds(truth)) print("WER: ", compute_WER(dataset.phoneme_tokenizer.DecodeIds(truth), dataset.phoneme_tokenizer.DecodeIds(guess))) print("") train_loss /= num_examples train_WER /= num_examples #self.model.unfreeze_one_layer() results = {"loss" : train_loss, "WER" : train_WER, "set": "train"} self.log(results) self.epoch += 1 return train_WER, train_loss
def train(self, dataset, print_interval=100, log_results=True): train_WER = 0 train_loss = 0 num_examples = 0 self.model.train() for g in self.optimizer.param_groups: print("Current learning rate:", g['lr']) FLOPs = [] for idx, batch in enumerate(tqdm(dataset.loader)): x, y, T, U, idxs = batch batch_size = len(x) log_probs, p_big, I_big = self.model( x, y, T, U, self.config.deterministic_train) batch_FLOPs = (1 - I_big.mean( )) * self.model.FLOPs_small + I_big.mean() * self.model.FLOPs_big FLOPs += [batch_FLOPs.item()] * batch_size loss = -log_probs.mean() if torch.isnan(loss): print("nan detected!") sys.exit() self.optimizer.zero_grad() loss.backward() clip_value = 5 torch.nn.utils.clip_grad_norm_(self.model.parameters(), clip_value) self.optimizer.step() train_loss += loss.item() * batch_size num_examples += batch_size if idx % print_interval == 0: print("loss: " + str(loss.cpu().data.numpy().item())) guesses, _, _ = self.model.infer( x, self.config.deterministic_train, T) guess = guesses[0][:U[0]] guess_decoded = self.remove_repeated_silence( dataset.tokenizer.DecodeIds(guess)) print("guess:", guess_decoded) truth = y[0].cpu().data.numpy().tolist()[:U[0]] truth_decoded = self.remove_repeated_silence( dataset.tokenizer.DecodeIds(truth)) print("truth:", truth_decoded) print("WER: ", compute_WER(truth_decoded, guess_decoded)) print("avg p_big: ", p_big.mean().item()) print("avg I_big: ", I_big.mean().item()) print("") train_loss /= num_examples train_WER /= num_examples FLOPs = np.array(FLOPs) FLOPs_mean = FLOPs.mean() FLOPs_std = FLOPs.std() results = { "loss": train_loss, "WER": train_WER, "FLOPs_mean": FLOPs_mean, "FLOPs_std": FLOPs_std, "set": "train", "surprisal-triggered": self.model.sample_based_on_surprisal_during_training } if log_results: self.log(results) self.epoch += 1 return train_WER, train_loss, FLOPs_mean, FLOPs_std