def train_rnn(args, train_dataset, validation_dataset):
model_save_path = get_path(args, "tf_rnn", create=True)
rnn = MDNRNN(args=args)
rnn.compile(optimizer=rnn.optimizer,
loss=rnn.loss_fn,
metrics=rnn.get_metrics())
print("Start training")
current_time = datetime.now().strftime("%Y%m%d-%H%M%S")
tensorboard_dir = model_save_path / "tensorboard" / current_time
rnn.fit(train_dataset,
validation_data=validation_dataset,
steps_per_epoch=args.rnn_epoch_steps,
epochs=args.rnn_num_steps // args.rnn_epoch_steps,
callbacks=[
tf.keras.callbacks.TensorBoard(log_dir=str(tensorboard_dir),
update_freq=20,
histogram_freq=1,
profile_batch=0),
tf.keras.callbacks.ModelCheckpoint(str(model_save_path /
"ckpt-e{epoch:03d}"),
verbose=1),
])
rnn.save(str(model_save_path))
print(f"Model saved to {model_save_path}")
separators=(',', ': '))
def random_batch():
indices = np.random.permutation(N_data)[0:args.rnn_batch_size]
# suboptimal b/c we are always only taking first set of steps
mu = data_mu[indices][:, :args.rnn_max_seq_len]
logvar = data_logvar[indices][:, :args.rnn_max_seq_len]
action = data_action[indices][:, :args.rnn_max_seq_len]
z = sample_vae(mu, logvar)
d = tf.cast(data_d[indices], tf.float16)[:, :args.rnn_max_seq_len]
return z, action, d
rnn = MDNRNN(args=args)
rnn.compile(optimizer=rnn.optimizer, loss=rnn.get_loss())
# train loop:
start = time.time()
step = 0
input_spec = tf.TensorSpec([1, args.max_frames, args.rnn_input_seq_width],
tf.float32)
rnn._set_inputs(input_spec)
tf.keras.models.save_model(rnn,
model_save_path,
include_optimizer=True,
save_format='tf')
for step in range(args.rnn_num_steps):
curr_learning_rate = (
args.rnn_learning_rate - args.rnn_min_learning_rate) * (