def run(self):
for epoch in range(self.start_epoch, self.epochs):
start = time.time()
train_loss, valid_loss = self.run_one_epoch(epoch)
end = time.time()
print("[Epoch {}/{}] loss (train): {:.5f}, loss (valid): {:.5f}, {:.3f} [sec]".format(epoch+1, self.epochs, train_loss, valid_loss, end - start), flush=True)
self.train_loss[epoch] = train_loss
self.valid_loss[epoch] = valid_loss
if (epoch + 1)%2 == 0:
optim_dict = self.optimizer.state_dict()
lr = optim_dict['param_groups'][0]['lr']
optim_dict['param_groups'][0]['lr'] = 0.98 * lr
self.optimizer.load_state_dict(optim_dict)
print("Learning rate: {} -> {}".format(lr, 0.98 * lr))
if valid_loss < self.best_loss:
self.best_loss = valid_loss
self.no_improvement = 0
model_path = os.path.join(self.model_dir, "best.pth")
self.save_model(epoch, model_path)
else:
self.no_improvement += 1
if self.no_improvement >= 10:
print("Stop training")
break
model_path = os.path.join(self.model_dir, "last.pth")
self.save_model(epoch, model_path)
save_path = os.path.join(self.loss_dir, "loss.png")
draw_loss_curve(train_loss=self.train_loss[:epoch+1], valid_loss=self.valid_loss[:epoch+1], save_path=save_path)
def run(self):
for epoch in range(self.start_epoch, self.epochs):
start = time.time()
train_loss = self.run_one_epoch(epoch)
end = time.time()
print("[Epoch {}/{}] loss (train): {:.5f}, {:.3f} [sec]".format(epoch+1, self.epochs, train_loss, end - start), flush=True)
self.train_loss[epoch] = train_loss
model_path = os.path.join(self.model_dir, "last.pth")
self.save_model(epoch, model_path)
save_path = os.path.join(self.loss_dir, "loss.png")
draw_loss_curve(train_loss=self.train_loss[:epoch+1], save_path=save_path)
def run(self):
for epoch in range(self.start_epoch, self.epochs):
start = time.time()
train_loss, valid_loss = self.run_one_epoch(epoch)
end = time.time()
print(
"[Epoch {}/{}] loss (train): {:.5f}, loss (valid): {:.5f}, {:.3f} [sec]"
.format(epoch + 1, self.epochs, train_loss, valid_loss,
end - start),
flush=True)
self.train_loss[epoch] = train_loss
self.valid_loss[epoch] = valid_loss
# Learning rate scheduling
# torch.optim.lr_scheduler.ExponentialLR may be useful.
lr_decay = self.lr_decay
optim_dict = self.optimizer.state_dict()
lr = optim_dict['param_groups'][0]['lr']
print("Learning rate: {} -> {}".format(lr, lr_decay * lr))
optim_dict['param_groups'][0]['lr'] = lr_decay * lr
self.optimizer.load_state_dict(optim_dict)
if valid_loss < self.best_loss:
self.best_loss = valid_loss
self.no_improvement = 0
model_path = os.path.join(self.model_dir, "best.pth")
self.save_model(epoch, model_path)
else:
if valid_loss >= self.prev_loss:
self.no_improvement += 1
if self.no_improvement >= 10:
print("Stop training")
break
else:
self.no_improvement = 0
self.prev_loss = valid_loss
model_path = os.path.join(self.model_dir, "last.pth")
self.save_model(epoch, model_path)
save_path = os.path.join(self.loss_dir, "loss.png")
draw_loss_curve(train_loss=self.train_loss[:epoch + 1],
valid_loss=self.valid_loss[:epoch + 1],
save_path=save_path)