def start_training(train_arguments, folder_index): rcnn = RCNN(train_arguments.pos_loss_method, train_arguments.loss_weight_lambda, train_arguments.prevent_overfitting_method).cuda() rcnn.train() # train mode could use dropout. npz_path = train_arguments.get_train_data_path(folder_index) npz = np.load(npz_path) print("\n\n\nload from: ", npz_path) train_arguments.train_sentences = npz['train_sentences'] train_arguments.train_sentence_info = npz['train_sentence_info'] train_arguments.train_roi = npz['train_roi'] train_arguments.train_cls = npz['train_cls'] if train_arguments.normalize: if train_arguments.dx_compute_method == "left_boundary": train_arguments.train_tbbox = npz["train_norm_lb_tbbox"] else: train_arguments.train_tbbox = npz["train_norm_tbbox"] else: train_arguments.train_tbbox = npz['train_tbbox'] train_arguments.train_sentences = t.Tensor(train_arguments.train_sentences) train_arguments.train_set = np.random.permutation(train_arguments.train_sentences.size(0)) # like shuffle if train_arguments.prevent_overfitting_method.lower() == "l2 regu": if train_arguments.partial_l2_penalty: optimizer = optim.Adam([ {"params": rcnn.conv1.parameters(), "weight_decay": 0}, {"params": rcnn.cls_fc1.parameters(), "weight_decay": train_arguments.l2_beta}, {"params": rcnn.cls_score.parameters(), "weight_decay": train_arguments.l2_beta}, {"params": rcnn.bbox_fc1.parameters(), "weight_decay": train_arguments.l2_beta}, {"params": rcnn.bbox.parameters(), "weight_decay": train_arguments.l2_beta} ], lr=train_arguments.learning_rate) else: optimizer = optim.Adam(rcnn.parameters(), lr=train_arguments.learning_rate, weight_decay=train_arguments.l2_beta) else: # dropout optimizer optimizer = optim.Adam(rcnn.parameters(), lr=train_arguments.learning_rate) rcnn.optimizer = optimizer for epoch_time in range(train_arguments.max_iter_epoch): print('===========================================') print('[Training Epoch {}]'.format(epoch_time + 1)) train_epoch(train_arguments, rcnn) if epoch_time >= train_arguments.start_save_epoch: save_directory = train_arguments.get_save_directory(folder_index) save_path = save_directory + "model_epoch" + str(epoch_time + 1) + ".pth" t.save(rcnn.state_dict(), save_path) print("Model save in ", save_path)
# Save it for later. with open(imagesdata_pickle_path, 'wb') as fi: pickle.dump(imagesdata, fi) print('NB CLASSES : ' + str(imagesdata.get_num_classes())) # check the classes tf.debugging.set_log_device_placement(True) strategy = tf.distribute.MirroredStrategy() # run on multiple GPUs with strategy.scope(): arch = RCNN(imagesdata, loss=loss, opt=opt, lr=lr, verbose=verbose) arch.train(epochs=epochs, batch_size=batch_size, split_size=split_size, checkpoint_path=checkpoint_path, early_stopping=early_stopping, verbose=verbose) arch.model.save(filepath='../data/out_test/test2/model.h5', save_format='h5') loss = arch.history()['loss'] val_loss = arch.history()['val_loss'] accuracy = arch.history()['accuracy'] val_accuracy = arch.history()['val_accuracy'] # Loss plt.plot(loss) plt.plot(val_loss) plt.title("Model loss")
if use_gpu: net = net.cuda() loss_trend = [] accuracy_trend = [] phase = 'train' for i in range(epoch): loss_each_epoch = [] running_accuracy = [] mini_count = 1 for images,labels in dataloader['train']: net = net.train() if use_gpu: images = images.cuda() labels = labels.cuda() optimizer.zero_grad() # scheduler.optimizer.zero_grad() output = net(images) # print(output) # print(target) loss = criterion(output, labels) loss.backward() optimizer.step() # scheduler.step(loss) loss_to_append = loss.clone().cpu().view(1).data.numpy()[0]