lr_decay = lr_decay_base ** max(epoch - m_flat_lr, 0)
lr = lr * lr_decay # decay lr if it is time
# RUN MODEL ON TRAINING DATA
train_ppl, train_loss = run_epoch(model, train_data, True, lr)
# RUN MODEL ON VALIDATION DATA
val_ppl, val_loss = run_epoch(model, valid_data)
# SAVE MODEL IF IT'S THE BEST SO FAR
if val_ppl < best_val_so_far:
best_val_so_far = val_ppl
if args.save_best:
print("Saving model parameters to best_params.pt")
torch.save(model.state_dict(), os.path.join(args.save_dir, 'best_params.pt'))
# NOTE ==============================================
# You will need to load these parameters into the same model
# for a couple Problems: so that you can compute the gradient
# of the loss w.r.t. hidden state as required in Problem 4.1
# and to sample from the the model as required in Problem 4.2
# We are not asking you to run on the test data, but if you
# want to look at test performance you would load the saved
# model and run on the test data with batch_size=1
# LOC RESULTS
train_ppls.append(train_ppl)
val_ppls.append(val_ppl)
train_losses.extend(train_loss)
val_losses.extend(val_loss)
times.append(time.time() - t0)
if args.optimizer == 'SGD_LR_SCHEDULE':
lr_decay = lr_decay_base**max(epoch - m_flat_lr, 0)
lr = lr * lr_decay # decay lr if it is time
# RUN MODEL ON TRAINING DATA
train_ppl, train_loss = run_epoch(model, train_data, True, lr)
# RUN MODEL ON VALIDATION DATA
val_ppl, val_loss = run_epoch(model, valid_data)
# SAVE MODEL IF IT'S THE BEST SO FAR
if val_ppl < best_val_so_far:
best_val_so_far = val_ppl
if args.save_best:
print("Saving model parameters to best_params.pt")
torch.save(model.state_dict(),
os.path.join(args.save_dir, 'best_params.pt'))
# NOTE ==============================================
# You will need to load these parameters into the same model
# for a couple Problems: so that you can compute the gradient
# of the loss w.r.t. hidden state as required in Problem 4.1
# and to sample from the the model as required in Problem 4.2
# We are not asking you to run on the test data, but if you
# want to look at test performance you would load the saved
# model and run on the test data with batch_size=1
# LOC RESULTS
train_ppls.append(train_ppl)
val_ppls.append(val_ppl)
train_losses.extend(train_loss)
val_losses.extend(val_loss)