f'{n_steps}-steps average loss: {sum(losses[-n_steps:]) / n_steps}',
pos=pos + 2)
summary_manager.display_loss(output, tag='Train')
summary_manager.display_scalar(tag='Meta/decoder_prenet_dropout',
scalar_value=model.decoder_prenet.rate)
summary_manager.display_scalar(tag='Meta/learning_rate',
scalar_value=model.optimizer.lr)
summary_manager.display_scalar(tag='Meta/reduction_factor',
scalar_value=model.r)
summary_manager.display_scalar(tag='Meta/drop_n_heads',
scalar_value=model.drop_n_heads)
if model.step % config['train_images_plotting_frequency'] == 0:
summary_manager.display_attention_heads(output,
tag='TrainAttentionHeads')
summary_manager.display_mel(mel=output['mel_linear'][0],
tag=f'Train/linear_mel_out')
summary_manager.display_mel(mel=output['final_output'][0],
tag=f'Train/predicted_mel')
residual = abs(output['mel_linear'] - output['final_output'])
summary_manager.display_mel(mel=residual[0],
tag=f'Train/conv-linear_residual')
summary_manager.display_mel(mel=mel[0], tag=f'Train/target_mel')
if model.step % config['weights_save_frequency'] == 0:
save_path = manager.save()
t.display(f'checkpoint at step {model.step}: {save_path}',
pos=len(config['n_steps_avg_losses']) + 2)
if model.step % config['validation_frequency'] == 0:
val_loss, time_taken = validate(model=model,
val_dataset=val_dataset,
if len(losses) > n_steps:
t.display(
f'{n_steps}-steps average loss: {sum(losses[-n_steps:]) / n_steps}',
pos=pos + 2)
summary_manager.display_loss(output, tag='Train')
summary_manager.display_scalar(tag='Meta/learning_rate',
scalar_value=model.optimizer.lr)
summary_manager.display_scalar(tag='Meta/decoder_prenet_dropout',
scalar_value=model.decoder_prenet.rate)
summary_manager.display_scalar(tag='Meta/drop_n_heads',
scalar_value=model.drop_n_heads)
if model.step % config['train_images_plotting_frequency'] == 0:
summary_manager.display_attention_heads(output,
tag='TrainAttentionHeads')
summary_manager.display_mel(mel=output['mel'][0],
tag=f'Train/predicted_mel')
summary_manager.display_mel(mel=mel[0], tag=f'Train/target_mel')
summary_manager.add_histogram(tag=f'Train/Predicted durations',
values=output['duration'])
summary_manager.add_histogram(tag=f'Train/Target durations',
values=durations)
if model.step % config['weights_save_frequency'] == 0:
save_path = manager.save()
t.display(f'checkpoint at step {model.step}: {save_path}',
pos=len(config['n_steps_avg_losses']) + 2)
if model.step % config['validation_frequency'] == 0:
t.display(f'Validating', pos=len(config['n_steps_avg_losses']) + 3)
val_loss, time_taken = validate(model=model,
val_dataset=val_dataset,