def start_training(train_arguments, folder_index):
rcnn = RCNN(train_arguments.pos_loss_method, train_arguments.loss_weight_lambda,
train_arguments.prevent_overfitting_method).cuda()
rcnn.train() # train mode could use dropout.
npz_path = train_arguments.get_train_data_path(folder_index)
npz = np.load(npz_path)
print("\n\n\nload from: ", npz_path)
train_arguments.train_sentences = npz['train_sentences']
train_arguments.train_sentence_info = npz['train_sentence_info']
train_arguments.train_roi = npz['train_roi']
train_arguments.train_cls = npz['train_cls']
if train_arguments.normalize:
if train_arguments.dx_compute_method == "left_boundary":
train_arguments.train_tbbox = npz["train_norm_lb_tbbox"]
else:
train_arguments.train_tbbox = npz["train_norm_tbbox"]
else:
train_arguments.train_tbbox = npz['train_tbbox']
train_arguments.train_sentences = t.Tensor(train_arguments.train_sentences)
train_arguments.train_set = np.random.permutation(train_arguments.train_sentences.size(0)) # like shuffle
if train_arguments.prevent_overfitting_method.lower() == "l2 regu":
if train_arguments.partial_l2_penalty:
optimizer = optim.Adam([
{"params": rcnn.conv1.parameters(), "weight_decay": 0},
{"params": rcnn.cls_fc1.parameters(), "weight_decay": train_arguments.l2_beta},
{"params": rcnn.cls_score.parameters(), "weight_decay": train_arguments.l2_beta},
{"params": rcnn.bbox_fc1.parameters(), "weight_decay": train_arguments.l2_beta},
{"params": rcnn.bbox.parameters(), "weight_decay": train_arguments.l2_beta}
], lr=train_arguments.learning_rate)
else:
optimizer = optim.Adam(rcnn.parameters(), lr=train_arguments.learning_rate,
weight_decay=train_arguments.l2_beta)
else: # dropout optimizer
optimizer = optim.Adam(rcnn.parameters(), lr=train_arguments.learning_rate)
rcnn.optimizer = optimizer
for epoch_time in range(train_arguments.max_iter_epoch):
print('===========================================')
print('[Training Epoch {}]'.format(epoch_time + 1))
train_epoch(train_arguments, rcnn)
if epoch_time >= train_arguments.start_save_epoch:
save_directory = train_arguments.get_save_directory(folder_index)
save_path = save_directory + "model_epoch" + str(epoch_time + 1) + ".pth"
t.save(rcnn.state_dict(), save_path)
print("Model save in ", save_path)
# Save it for later.
with open(imagesdata_pickle_path, 'wb') as fi:
pickle.dump(imagesdata, fi)
print('NB CLASSES : ' + str(imagesdata.get_num_classes())) # check the classes
tf.debugging.set_log_device_placement(True)
strategy = tf.distribute.MirroredStrategy() # run on multiple GPUs
with strategy.scope():
arch = RCNN(imagesdata, loss=loss, opt=opt, lr=lr, verbose=verbose)
arch.train(epochs=epochs,
batch_size=batch_size,
split_size=split_size,
checkpoint_path=checkpoint_path,
early_stopping=early_stopping,
verbose=verbose)
arch.model.save(filepath='../data/out_test/test2/model.h5',
save_format='h5')
loss = arch.history()['loss']
val_loss = arch.history()['val_loss']
accuracy = arch.history()['accuracy']
val_accuracy = arch.history()['val_accuracy']
# Loss
plt.plot(loss)
plt.plot(val_loss)
plt.title("Model loss")
if use_gpu: net = net.cuda() loss_trend = [] accuracy_trend = [] phase = 'train' for i in range(epoch): loss_each_epoch = [] running_accuracy = [] mini_count = 1 for images,labels in dataloader['train']: net = net.train() if use_gpu: images = images.cuda() labels = labels.cuda() optimizer.zero_grad() # scheduler.optimizer.zero_grad() output = net(images) # print(output) # print(target) loss = criterion(output, labels) loss.backward() optimizer.step() # scheduler.step(loss) loss_to_append = loss.clone().cpu().view(1).data.numpy()[0]
