def train_model(model, device, optimizer, scheduler, train_loader, valid_loader, epochs, update_batch, model_name, save_dir, log_file): """ Train a deep neural network model Args: model : pytorch model object device : cuda or cpu optimizer : pytorch optimizer object scheduler : learning rate scheduler object that wraps the optimizer train_dataloader : training images dataloader valid_dataloader : validation images dataloader epochs : number of training epochs update_batch : after how many batches should gradient update be performed save_dir : Location to save model weights, plots and log_file log_file : text file instance to record training and validation history Returns: Training history and Validation history (loss) """ tr_loss = [] valid_loss = [] best_val_loss = np.iinfo(np.int32).max weights_path = os.path.join(save_dir, model_name) temp_weights_path = os.path.join(save_dir, "temp_{}".format(model_name)) last_batch = math.ceil(len(train_loader.dataset) / update_batch) # Each epoch has a training and validation phase for epoch in range(epochs): print("-------Epoch {}----------".format(epoch + 1)) log_file.write("-------Epoch {}----------".format(epoch + 1)) criterion = TripletLoss(margin=2) train_loader.dataset.reset() if (epoch + 1) % update_batch == 0: print("> Modifying learning rate") scheduler.step() for phase in ['train', 'valid']: running_loss = 0.0 if phase == 'train': model.train(True) # Set model to training mode # zero the parameter gradients optimizer.zero_grad() print("> Training the network") for batch_idx, [anchor, positive, negative] in enumerate(tqdm(train_loader)): anchor, positive, negative = anchor.to( device), positive.to(device), negative.to(device) # Retrieve the embeddings output1, output2, output3 = model(anchor, positive, negative) # Calculate the triplet loss loss = criterion(output1, output2, output3) # Sum up batch loss running_loss += loss # Backpropagate the system the determine the gradients loss.backward() if (batch_idx + 1) % update_batch == 0 or ( batch_idx + 1) == last_batch: # Update the paramteres of the model optimizer.step() # zero the parameter gradients optimizer.zero_grad() # clear variables del anchor, positive, negative, output1, output2, output3 gc.collect() torch.cuda.empty_cache() # Save trained model print("> Saving trained weights...") torch.save(model.state_dict(), temp_weights_path) # Calculate statistics and log num_samples = float(len(train_loader.dataset)) tr_loss_ = running_loss.item() / num_samples tr_loss.append(tr_loss_) print('> train_loss: {:.4f}\t'.format(tr_loss_)) log_file.write('> train_loss: {:.4f}\t'.format(tr_loss_)) else: model.train(False) print("> Running validation on the network") with torch.no_grad(): for anchor, positive, negative in tqdm(valid_loader): anchor, positive, negative = anchor.to( device), positive.to(device), negative.to(device) # Retrieve the embeddings output1, output2, output3 = model( anchor, positive, negative) # Calculate the triplet loss loss = criterion(output1, output2, output3) # Sum up batch loss running_loss += loss # clear variables del anchor, positive, negative, output1, output2, output3 gc.collect() torch.cuda.empty_cache() # Get statistics and log num_samples = float(len(valid_loader.dataset)) valid_loss_ = running_loss.item() / num_samples valid_loss.append(valid_loss_) print('> valid_loss: {:.4f}\t'.format(valid_loss_)) log_file.write('> valid_loss: {:.4f}\t'.format(valid_loss_)) if valid_loss_ < best_val_loss: best_val_loss = valid_loss_ print("> Saving best weights...") log_file.write("Saving best weights...\n") torch.save(model.state_dict(), weights_path) return (tr_loss, valid_loss)
flags.DEFINE_integer('batch_size', 150, 'Batch size.') flags.DEFINE_integer('train_iter', 100, 'Total training iter') flags.DEFINE_integer('step', 50, 'Save after ... iteration') flags.DEFINE_float('learning_rate', '0.01', 'Learning rate') flags.DEFINE_float('momentum', '0.99', 'Momentum') flags.DEFINE_string('model', 'conv_net', 'model to run') flags.DEFINE_string('data_src', r'C:\OData', 'source of training dataset') #Low_data OData all tf.compat.v1.disable_eager_execution() if __name__ == "__main__": # Setup Dataset dataset = PreProcessing(FLAGS.data_src) model = TripletLoss() placeholder_shape = [None] + list(dataset.images_train.shape[1:]) print("placeholder_shape", placeholder_shape) # Setup Network next_batch = dataset.get_triplets_batch anchor_input = tf.placeholder(tf.float32, placeholder_shape, name='anchor_input') positive_input = tf.placeholder(tf.float32, placeholder_shape, name='positive_input') negative_input = tf.placeholder(tf.float32, placeholder_shape, name='negative_input')
def train(): data_src = './ChallengeImages2/' learning_rate = 0.01 train_iter = 250 batch_size = 128 momentum = 0.99 step = 50 # Setup Dataset dataset = PreProcessing(data_src) print(dataset.images_test) print(dataset.labels_test) model = TripletLoss() height = 50 width = 150 dims = [height, width, 3] placeholder_shape = [None] + dims print("placeholder_shape", placeholder_shape) # Setup Network next_batch = dataset.get_triplets_batch anchor_input = tf.placeholder(tf.float32, placeholder_shape, name='anchor_input') positive_input = tf.placeholder(tf.float32, placeholder_shape, name='positive_input') negative_input = tf.placeholder(tf.float32, placeholder_shape, name='negative_input') margin = 0.5 # Will be of size N x 28 anchor_output = model.conv_net(anchor_input, reuse=tf.AUTO_REUSE) positive_output = model.conv_net(positive_input, reuse=tf.AUTO_REUSE) negative_output = model.conv_net(negative_input, reuse=tf.AUTO_REUSE) print(tf.shape(anchor_output)) return ''' compute the similarity between the positive_output and the negative_output if similarity is < 0.25 that's great ''' similarity_pos_neg = computeCosSimilarity(positive_output, negative_output) similarity_pos_anchor = computeCosSimilarity(positive_output, anchor_output) loss = model.triplet_loss(anchor_output, positive_output, negative_output, margin) # Setup Optimizer global_step = tf.Variable(0, trainable=False) train_step = tf.train.MomentumOptimizer(learning_rate, momentum, use_nesterov=True).minimize( loss, global_step=global_step) # Start Training saver = tf.train.Saver() with tf.Session() as sess: sess.run(tf.global_variables_initializer()) # Setup Tensorboard tf.summary.scalar('step', global_step) tf.summary.scalar('loss', loss) for var in tf.trainable_variables(): tf.summary.histogram(var.op.name, var) merged = tf.summary.merge_all() writer = tf.summary.FileWriter('train.log', sess.graph) # Train iter for i in range(train_iter): batch_anchor, batch_positive, batch_negative = next_batch( batch_size) _, l, summary_str = sess.run( [train_step, loss, merged], feed_dict={ anchor_input: batch_anchor, positive_input: batch_positive, negative_input: batch_negative }) #pNAccuracy = computeAccuracyLess(0.25, similarity_p_n) #pAAccuracy = computeAccuracyGreat(0.75, similarity_p_a) writer.add_summary(summary_str, i) print("#%d - Loss" % i, l) #print("#%d - P and A Accuracy" % i, pAAccuracy) #print("#%d - N and A Accuracy" % i, pNAccuracy) if (i + 1) % step == 0: saver.save(sess, "model_triplet/model.ckpt") saver.save(sess, "model_triplet/model.ckpt") print('Training completed successfully.') return dataset
Y = np.asarray([label_dict[label] for label in y]) shuffle_indices = np.random.permutation(np.arange(len(y))) x_shuffled = [] y_shuffled = [] for index in shuffle_indices: x_shuffled.append(X[index]) y_shuffled.append(Y[index]) size_of_dataset = len(x_shuffled) n_train = int(np.ceil(size_of_dataset * train_test_ratio)) n_test = int(np.ceil(size_of_dataset * (1 - train_test_ratio))) return np.asarray(x_shuffled[0:n_train]), np.asarray(x_shuffled[n_train + 1:size_of_dataset]), np.asarray(y_shuffled[0:n_train]), np.asarray(y_shuffled[n_train + 1:size_of_dataset]) train_images, test_images, train_label, test_label = get_train_test_dataset(0.7) model_path = './model_triplet/' model = TripletLoss() # Input and output tensor img_placeholder = tf.placeholder(tf.float32, [None, 28, 28, 3], name='img') net = model.conv_net(img_placeholder, reuse=False) print(net) # generate random index from test image corpus and display the image idx = np.random.randint(0, len(test_images)) im = test_images[idx] # show the test image print('************Query Image**************') #show_image(idx, test_images) # Find k-nearest neighbor using cosine similarity
tf.flags.DEFINE_integer('max_document_length',15,'maximum number of word in a sentence') tf.flags.DEFINE_float('momentum','0.99', 'Momentum') tf.flags.DEFINE_string('model', 'conv_net', 'model to run') tf.flags.DEFINE_string('data_src', './data_repository/questions.csv', 'source of training dataset') flags = tf.app.flags FLAGS = flags.FLAGS if __name__ == "__main__": # Setup Dataset dataset = PreProcessing(FLAGS.data_src, FLAGS.max_document_length) model = TripletLoss(sequence_length=dataset.X.shape[1], vocab_size=len(dataset.vocab_processor.vocabulary_), embedding_size=FLAGS.embedding_dim, filter_sizes=list(map(int, FLAGS.filter_sizes.split(","))), num_filters=FLAGS.num_filters, output_embedding_size = FLAGS.output_embedding_size, dropout_keep_prob = FLAGS.dropout_keep_prob, embeddings_lookup= dataset.embeddings_lookup, l2_reg_lambda=FLAGS.l2_reg_lambda) placeholder_shape = [None] + [dataset.X.shape[1]] print("placeholder_shape", placeholder_shape) # Setup Network next_batch = dataset.get_triplets_batch anchor_input = tf.placeholder(tf.int32, placeholder_shape, name='anchor_input') positive_input = tf.placeholder(tf.int32, placeholder_shape, name='positive_input') negative_input = tf.placeholder(tf.int32, placeholder_shape, name='negative_input') margin = 3.5 anchor_output = model.conv_net(anchor_input, reuse=False)