iter_start_time = time.time()
if total_steps % opt.print_freq == 0:
t_data = iter_start_time - iter_data_time
total_steps += opt.batch_size
epoch_iter += opt.batch_size
model.set_input(data)
model.optimize_parameters()
loss_mat.append(model.loss.cpu().data.numpy())
#CE_mat.append(model.CE_loss.cpu().data.numpy())
#prior_mat.append(model.prior_loss.cpu().data.numpy())
if total_steps % opt.print_freq == 0:
loss = model.loss
t = (time.time() - iter_start_time) / opt.batch_size
writer.print_current_losses(epoch, epoch_iter, loss, t, t_data)
writer.plot_loss(loss, epoch, epoch_iter, dataset_size)
if i % opt.save_latest_freq == 0:
print('saving the latest model (epoch %d, total_steps %d)' %
(epoch, total_steps))
model.save_network('latest')
iter_data_time = time.time()
writer.save_losses(loss_mat, CE_mat, prior_mat, epoch)
if epoch % opt.save_epoch_freq == 0:
print('saving the model at the end of epoch %d, iters %d' %
(epoch, total_steps))
model.save_network('latest')
model.save_network(epoch)
print('End of epoch %d / %d \t Time Taken: %d sec' %
start_time = time.time()
count = 0
running_loss = 0.0
for i, data in enumerate(loader):
# break
if data.y.size(0) % 64 != 0:
continue
total_steps += opt.batch_size
count += opt.batch_size
model.set_input_data(data)
model.optimize()
running_loss += model.loss_val
if total_steps % opt.frequency == 0:
loss_val = running_loss / opt.frequency
writer.print_loss(epoch, count, loss_val)
writer.plot_loss(epoch, count, loss_val, len(dataset))
running_loss = 0
if i % opt.loop_frequency == 0:
print('saving the latest model (epoch %d, total_steps %d)' %
(epoch, total_steps))
model.save_network('latest')
# break
if epoch % opt.epoch_frequency == 0:
print('saving the model at the end of epoch %d, iters %d' %
(epoch, total_steps))
if (epoch - 1) % 20 == 0:
model.log_history_and_plot(writer, epoch, count)
model.log_features_and_plot(epoch, count)
model.save_network('latest')
continue
if total_steps % opt.print_freq == 0:
loss = model.loss
t = (time.time() - iter_start_time) / opt.batch_size
writer.print_current_losses(epoch, epoch_iter, loss, t, t_data)
#writer.plot_loss(loss, epoch, epoch_iter, dataset_size)
if i % opt.save_latest_freq == 0:
print('saving the latest model (epoch %d, total_steps %d)' %
(epoch, total_steps))
model.save_network('latest_net')
iter_data_time = time.time()
writer.plot_loss(loss, epoch, 1, 1)
print('TRAIN ACC [%.3f]' % (writer.acc))
writer.plot_train_acc(writer.acc, epoch)
if epoch % opt.save_epoch_freq == 0:
print('saving the model at the end of epoch %d, iters %d' %
(epoch, total_steps))
model.save_network('latest_net')
# model.save_network('epoch_%d' % (epoch))
model.update_learning_rate()
if opt.verbose_plot:
writer.plot_model_wts(model, epoch)
if epoch % opt.run_test_freq == 0:
acc = run_test(epoch)
