def test_model():
"""Evaluates the model."""
# Build the model (before the loaders to speed up debugging)
model = model_builder.build_model()
log_model_info(model)
# Compute precise time
if cfg.PREC_TIME.ENABLED:
logger.info("Computing precise time...")
loss_fun = losses.get_loss_fun()
bu.compute_precise_time(model, loss_fun)
nu.reset_bn_stats(model)
# Load model weights
cu.load_checkpoint(cfg.TEST.WEIGHTS, model)
logger.info("Loaded model weights from: {}".format(cfg.TEST.WEIGHTS))
# Create data loaders
test_loader = loader.construct_test_loader()
# Create meters
test_meter = TestMeter(len(test_loader))
# Evaluate the model
test_epoch(test_loader, model, test_meter, 0)
def train_model():
"""Trains the model."""
# Build the model (before the loaders to speed up debugging)
model = model_builder.build_model()
log_model_info(model)
# Define the loss function
loss_fun = losses.get_loss_fun()
# Construct the optimizer
optimizer = optim.construct_optimizer(model)
# Load checkpoint or initial weights
start_epoch = 0
if cfg.TRAIN.AUTO_RESUME and cu.has_checkpoint():
last_checkpoint = cu.get_last_checkpoint()
checkpoint_epoch = cu.load_checkpoint(last_checkpoint, model,
optimizer)
logger.info("Loaded checkpoint from: {}".format(last_checkpoint))
start_epoch = checkpoint_epoch + 1
elif cfg.TRAIN.WEIGHTS:
cu.load_checkpoint(cfg.TRAIN.WEIGHTS, model)
logger.info("Loaded initial weights from: {}".format(
cfg.TRAIN.WEIGHTS))
# Compute precise time
if start_epoch == 0 and cfg.PREC_TIME.ENABLED:
logger.info("Computing precise time...")
bu.compute_precise_time(model, loss_fun)
nu.reset_bn_stats(model)
# Create data loaders
train_loader = loader.construct_train_loader()
test_loader = loader.construct_test_loader()
# Create meters
train_meter = TrainMeter(len(train_loader))
test_meter = TestMeter(len(test_loader))
# Perform the training loop
logger.info("Start epoch: {}".format(start_epoch + 1))
for cur_epoch in range(start_epoch, cfg.OPTIM.MAX_EPOCH):
# Train for one epoch
train_epoch(train_loader, model, loss_fun, optimizer, train_meter,
cur_epoch)
# Compute precise BN stats
if cfg.BN.USE_PRECISE_STATS:
nu.compute_precise_bn_stats(model, train_loader)
# Save a checkpoint
if cu.is_checkpoint_epoch(cur_epoch):
checkpoint_file = cu.save_checkpoint(model, optimizer, cur_epoch)
logger.info("Wrote checkpoint to: {}".format(checkpoint_file))
# Evaluate the model
if is_eval_epoch(cur_epoch):
test_epoch(test_loader, model, test_meter, cur_epoch)
def ensemble_train_model(train_loader, val_loader, model, optimizer, cfg):
global plot_epoch_xvalues
global plot_epoch_yvalues
global plot_it_x_values
global plot_it_y_values
start_epoch = 0
loss_fun = losses.get_loss_fun()
# Create meters
train_meter = TrainMeter(len(train_loader))
val_meter = ValMeter(len(val_loader))
# Perform the training loop
# print("Len(train_loader):{}".format(len(train_loader)))
logger.info('Start epoch: {}'.format(start_epoch + 1))
val_set_acc = 0.
temp_best_val_acc = 0.
temp_best_val_epoch = 0
# Best checkpoint model and optimizer states
best_model_state = None
best_opt_state = None
val_acc_epochs_x = []
val_acc_epochs_y = []
clf_train_iterations = cfg.OPTIM.MAX_EPOCH * int(
len(train_loader) / cfg.TRAIN.BATCH_SIZE)
clf_change_lr_iter = clf_train_iterations // 25
clf_iter_count = 0
for cur_epoch in range(start_epoch, cfg.OPTIM.MAX_EPOCH):
# Train for one epoch
train_loss, clf_iter_count = train_epoch(train_loader, model, loss_fun, optimizer, train_meter, \
cur_epoch, cfg, clf_iter_count, clf_change_lr_iter, clf_train_iterations)
# Compute precise BN stats
if cfg.BN.USE_PRECISE_STATS:
nu.compute_precise_bn_stats(model, train_loader)
# Model evaluation
if is_eval_epoch(cur_epoch):
# Original code[PYCLS] passes on testLoader but we want to compute on val Set
val_set_err = test_epoch(val_loader, model, val_meter, cur_epoch)
val_set_acc = 100. - val_set_err
if temp_best_val_acc < val_set_acc:
temp_best_val_acc = val_set_acc
temp_best_val_epoch = cur_epoch + 1
# Save best model and optimizer state for checkpointing
model.eval()
best_model_state = model.module.state_dict(
) if cfg.NUM_GPUS > 1 else model.state_dict()
best_opt_state = optimizer.state_dict()
model.train()
# Since we start from 0 epoch
val_acc_epochs_x.append(cur_epoch + 1)
val_acc_epochs_y.append(val_set_acc)
plot_epoch_xvalues.append(cur_epoch + 1)
plot_epoch_yvalues.append(train_loss)
save_plot_values([plot_epoch_xvalues, plot_epoch_yvalues, plot_it_x_values, plot_it_y_values, val_acc_epochs_x, val_acc_epochs_y],\
["plot_epoch_xvalues", "plot_epoch_yvalues", "plot_it_x_values", "plot_it_y_values","val_acc_epochs_x","val_acc_epochs_y"], out_dir=cfg.EPISODE_DIR, isDebug=False)
logger.info("Successfully logged numpy arrays!!")
# Plot arrays
plot_arrays(x_vals=plot_epoch_xvalues, y_vals=plot_epoch_yvalues, \
x_name="Epochs", y_name="Loss", dataset_name=cfg.DATASET.NAME, out_dir=cfg.EPISODE_DIR)
plot_arrays(x_vals=val_acc_epochs_x, y_vals=val_acc_epochs_y, \
x_name="Epochs", y_name="Validation Accuracy", dataset_name=cfg.DATASET.NAME, out_dir=cfg.EPISODE_DIR)
save_plot_values([plot_epoch_xvalues, plot_epoch_yvalues, plot_it_x_values, plot_it_y_values, val_acc_epochs_x, val_acc_epochs_y], \
["plot_epoch_xvalues", "plot_epoch_yvalues", "plot_it_x_values", "plot_it_y_values","val_acc_epochs_x","val_acc_epochs_y"], out_dir=cfg.EPISODE_DIR)
print('Training Epoch: {}/{}\tTrain Loss: {}\tVal Accuracy: {}'.format(
cur_epoch + 1, cfg.OPTIM.MAX_EPOCH, round(train_loss, 4),
round(val_set_acc, 4)))
# Save the best model checkpoint (Episode level)
checkpoint_file = cu.save_checkpoint(info="vlBest_acc_"+str(int(temp_best_val_acc)), \
model_state=best_model_state, optimizer_state=best_opt_state, epoch=temp_best_val_epoch, cfg=cfg)
print('\nWrote Best Model Checkpoint to: {}\n'.format(
checkpoint_file.split('/')[-1]))
logger.info('Wrote Best Model Checkpoint to: {}\n'.format(checkpoint_file))
plot_arrays(x_vals=plot_epoch_xvalues, y_vals=plot_epoch_yvalues, \
x_name="Epochs", y_name="Loss", dataset_name=cfg.DATASET.NAME, out_dir=cfg.EPISODE_DIR)
plot_arrays(x_vals=plot_it_x_values, y_vals=plot_it_y_values, \
x_name="Iterations", y_name="Loss", dataset_name=cfg.DATASET.NAME, out_dir=cfg.EPISODE_DIR)
plot_arrays(x_vals=val_acc_epochs_x, y_vals=val_acc_epochs_y, \
x_name="Epochs", y_name="Validation Accuracy", dataset_name=cfg.DATASET.NAME, out_dir=cfg.EPISODE_DIR)
plot_epoch_xvalues = []
plot_epoch_yvalues = []
plot_it_x_values = []
plot_it_y_values = []
best_val_acc = temp_best_val_acc
best_val_epoch = temp_best_val_epoch
return best_val_acc, best_val_epoch, checkpoint_file