def test(self, dataset, set):
test_WER = 0
test_loss = 0
num_examples = 0
self.model.eval()
for idx, batch in enumerate(tqdm(dataset.loader)): #enumerate(dataset.loader):
x,y,T,U,_ = batch
batch_size = len(x)
num_examples += batch_size
log_probs = self.model(x,y,T,U)
loss = -log_probs.mean()
test_loss += loss.item() * batch_size
WERs = []
guesses = self.model.infer(x[:2], T[:2])
print("decoding subset")
for i in range(2): #range(batch_size):
guess = guesses[i][:U[i]]
truth = y[i].cpu().data.numpy().tolist()[:U[i]]
WERs.append(compute_WER(dataset.phoneme_tokenizer.DecodeIds(truth), dataset.phoneme_tokenizer.DecodeIds(guess)))
WER = np.array(WERs).mean()
test_WER += WER * batch_size
print("guess:", dataset.phoneme_tokenizer.DecodeIds(guess))
print("truth:", dataset.phoneme_tokenizer.DecodeIds(truth))
print("WER: ", compute_WER(dataset.phoneme_tokenizer.DecodeIds(truth), dataset.phoneme_tokenizer.DecodeIds(guess)))
print("")
test_loss /= num_examples
self.scheduler.step()
test_WER /= num_examples
results = {"loss" : test_loss, "WER" : test_WER, "set": set}
self.log(results)
return test_WER, test_loss
def test(self, dataset, set, log_results=True, print_results=True):
test_WER = 0
test_loss = 0
num_examples = 0
self.model.eval()
FLOPs = []
for idx, batch in enumerate(dataset.loader):
x, y, T, U, _ = batch
batch_size = len(x)
num_examples += batch_size
log_probs, _, _ = self.model(x, y, T, U,
self.config.deterministic_test)
loss = -log_probs.mean()
test_loss += loss.item() * batch_size
WERs = []
guesses, p_big, I_big = self.model.infer(
x, self.config.deterministic_test, T)
batch_FLOPs = (1 - I_big.mean(
)) * self.model.FLOPs_small + I_big.mean() * self.model.FLOPs_big
FLOPs += [batch_FLOPs.item()] * batch_size
for i in range(batch_size):
guess = guesses[i][:U[i]]
truth = y[i].cpu().data.numpy().tolist()[:U[i]]
guess_decoded = self.remove_repeated_silence(
dataset.tokenizer.DecodeIds(guess))
truth_decoded = self.remove_repeated_silence(
dataset.tokenizer.DecodeIds(truth))
WERs.append(compute_WER(truth_decoded, guess_decoded))
WER = np.array(WERs).mean()
test_WER += WER * batch_size
if print_results:
print("guess:", guess_decoded)
print("truth:", truth_decoded)
print("WER: ", compute_WER(truth_decoded, guess_decoded))
print("p_big: ", p_big.mean().item())
print("")
test_loss /= num_examples
self.scheduler.step()
test_WER /= num_examples
FLOPs = np.array(FLOPs)
FLOPs_mean = FLOPs.mean()
FLOPs_std = FLOPs.std()
results = {
"loss":
test_loss,
"WER":
test_WER,
"FLOPs_mean":
FLOPs_mean,
"FLOPs_std":
FLOPs_std,
"set":
set,
"surprisal-triggered":
self.model.sample_based_on_surprisal_during_testing
}
if log_results: self.log(results)
return test_WER, test_loss, FLOPs_mean, FLOPs_std
def train(self, dataset, print_interval=100):
train_WER = 0
train_loss = 0
num_examples = 0
self.model.train()
for g in self.optimizer.param_groups:
print("Current learning rate:", g['lr'])
#self.model.print_frozen()
for idx, batch in enumerate(tqdm(dataset.loader)):
x,y,T,U,idxs = batch
batch_size = len(x)
log_probs = self.model(x,y,T,U)
loss = -log_probs.mean()
if torch.isnan(loss):
print("nan detected!")
sys.exit()
self.optimizer.zero_grad()
loss.backward()
clip_value = 5; torch.nn.utils.clip_grad_norm_(self.model.parameters(), clip_value)
self.optimizer.step()
train_loss += loss.item() * batch_size
num_examples += batch_size
if idx % print_interval == 0:
print("loss: " + str(loss.cpu().data.numpy().item()))
guess = self.model.infer(x[:1], T[:1])[0][:U[0]]
print("guess:", dataset.phoneme_tokenizer.DecodeIds(guess))
truth = y[0].cpu().data.numpy().tolist()[:U[0]]
print("truth:", dataset.phoneme_tokenizer.DecodeIds(truth))
print("WER: ", compute_WER(dataset.phoneme_tokenizer.DecodeIds(truth), dataset.phoneme_tokenizer.DecodeIds(guess)))
print("")
train_loss /= num_examples
train_WER /= num_examples
#self.model.unfreeze_one_layer()
results = {"loss" : train_loss, "WER" : train_WER, "set": "train"}
self.log(results)
self.epoch += 1
return train_WER, train_loss
def train(self, dataset, print_interval=100, log_results=True):
train_WER = 0
train_loss = 0
num_examples = 0
self.model.train()
for g in self.optimizer.param_groups:
print("Current learning rate:", g['lr'])
FLOPs = []
for idx, batch in enumerate(tqdm(dataset.loader)):
x, y, T, U, idxs = batch
batch_size = len(x)
log_probs, p_big, I_big = self.model(
x, y, T, U, self.config.deterministic_train)
batch_FLOPs = (1 - I_big.mean(
)) * self.model.FLOPs_small + I_big.mean() * self.model.FLOPs_big
FLOPs += [batch_FLOPs.item()] * batch_size
loss = -log_probs.mean()
if torch.isnan(loss):
print("nan detected!")
sys.exit()
self.optimizer.zero_grad()
loss.backward()
clip_value = 5
torch.nn.utils.clip_grad_norm_(self.model.parameters(), clip_value)
self.optimizer.step()
train_loss += loss.item() * batch_size
num_examples += batch_size
if idx % print_interval == 0:
print("loss: " + str(loss.cpu().data.numpy().item()))
guesses, _, _ = self.model.infer(
x, self.config.deterministic_train, T)
guess = guesses[0][:U[0]]
guess_decoded = self.remove_repeated_silence(
dataset.tokenizer.DecodeIds(guess))
print("guess:", guess_decoded)
truth = y[0].cpu().data.numpy().tolist()[:U[0]]
truth_decoded = self.remove_repeated_silence(
dataset.tokenizer.DecodeIds(truth))
print("truth:", truth_decoded)
print("WER: ", compute_WER(truth_decoded, guess_decoded))
print("avg p_big: ", p_big.mean().item())
print("avg I_big: ", I_big.mean().item())
print("")
train_loss /= num_examples
train_WER /= num_examples
FLOPs = np.array(FLOPs)
FLOPs_mean = FLOPs.mean()
FLOPs_std = FLOPs.std()
results = {
"loss":
train_loss,
"WER":
train_WER,
"FLOPs_mean":
FLOPs_mean,
"FLOPs_std":
FLOPs_std,
"set":
"train",
"surprisal-triggered":
self.model.sample_based_on_surprisal_during_training
}
if log_results: self.log(results)
self.epoch += 1
return train_WER, train_loss, FLOPs_mean, FLOPs_std