def train_model(model, device, optimizer, scheduler, train_loader,
valid_loader, epochs, update_batch, model_name, save_dir,
log_file):
"""
Train a deep neural network model
Args:
model : pytorch model object
device : cuda or cpu
optimizer : pytorch optimizer object
scheduler : learning rate scheduler object that wraps the optimizer
train_dataloader : training images dataloader
valid_dataloader : validation images dataloader
epochs : number of training epochs
update_batch : after how many batches should gradient update be performed
save_dir : Location to save model weights, plots and log_file
log_file : text file instance to record training and validation history
Returns:
Training history and Validation history (loss)
"""
tr_loss = []
valid_loss = []
best_val_loss = np.iinfo(np.int32).max
weights_path = os.path.join(save_dir, model_name)
temp_weights_path = os.path.join(save_dir, "temp_{}".format(model_name))
last_batch = math.ceil(len(train_loader.dataset) / update_batch)
# Each epoch has a training and validation phase
for epoch in range(epochs):
print("-------Epoch {}----------".format(epoch + 1))
log_file.write("-------Epoch {}----------".format(epoch + 1))
criterion = TripletLoss(margin=2)
train_loader.dataset.reset()
if (epoch + 1) % update_batch == 0:
print("> Modifying learning rate")
scheduler.step()
for phase in ['train', 'valid']:
running_loss = 0.0
if phase == 'train':
model.train(True) # Set model to training mode
# zero the parameter gradients
optimizer.zero_grad()
print("> Training the network")
for batch_idx, [anchor, positive,
negative] in enumerate(tqdm(train_loader)):
anchor, positive, negative = anchor.to(
device), positive.to(device), negative.to(device)
# Retrieve the embeddings
output1, output2, output3 = model(anchor, positive,
negative)
# Calculate the triplet loss
loss = criterion(output1, output2, output3)
# Sum up batch loss
running_loss += loss
# Backpropagate the system the determine the gradients
loss.backward()
if (batch_idx + 1) % update_batch == 0 or (
batch_idx + 1) == last_batch:
# Update the paramteres of the model
optimizer.step()
# zero the parameter gradients
optimizer.zero_grad()
# clear variables
del anchor, positive, negative, output1, output2, output3
gc.collect()
torch.cuda.empty_cache()
# Save trained model
print("> Saving trained weights...")
torch.save(model.state_dict(), temp_weights_path)
# Calculate statistics and log
num_samples = float(len(train_loader.dataset))
tr_loss_ = running_loss.item() / num_samples
tr_loss.append(tr_loss_)
print('> train_loss: {:.4f}\t'.format(tr_loss_))
log_file.write('> train_loss: {:.4f}\t'.format(tr_loss_))
else:
model.train(False)
print("> Running validation on the network")
with torch.no_grad():
for anchor, positive, negative in tqdm(valid_loader):
anchor, positive, negative = anchor.to(
device), positive.to(device), negative.to(device)
# Retrieve the embeddings
output1, output2, output3 = model(
anchor, positive, negative)
# Calculate the triplet loss
loss = criterion(output1, output2, output3)
# Sum up batch loss
running_loss += loss
# clear variables
del anchor, positive, negative, output1, output2, output3
gc.collect()
torch.cuda.empty_cache()
# Get statistics and log
num_samples = float(len(valid_loader.dataset))
valid_loss_ = running_loss.item() / num_samples
valid_loss.append(valid_loss_)
print('> valid_loss: {:.4f}\t'.format(valid_loss_))
log_file.write('> valid_loss: {:.4f}\t'.format(valid_loss_))
if valid_loss_ < best_val_loss:
best_val_loss = valid_loss_
print("> Saving best weights...")
log_file.write("Saving best weights...\n")
torch.save(model.state_dict(), weights_path)
return (tr_loss, valid_loss)
flags.DEFINE_integer('batch_size', 150, 'Batch size.')
flags.DEFINE_integer('train_iter', 100, 'Total training iter')
flags.DEFINE_integer('step', 50, 'Save after ... iteration')
flags.DEFINE_float('learning_rate', '0.01', 'Learning rate')
flags.DEFINE_float('momentum', '0.99', 'Momentum')
flags.DEFINE_string('model', 'conv_net', 'model to run')
flags.DEFINE_string('data_src', r'C:\OData',
'source of training dataset') #Low_data OData all
tf.compat.v1.disable_eager_execution()
if __name__ == "__main__":
# Setup Dataset
dataset = PreProcessing(FLAGS.data_src)
model = TripletLoss()
placeholder_shape = [None] + list(dataset.images_train.shape[1:])
print("placeholder_shape", placeholder_shape)
# Setup Network
next_batch = dataset.get_triplets_batch
anchor_input = tf.placeholder(tf.float32,
placeholder_shape,
name='anchor_input')
positive_input = tf.placeholder(tf.float32,
placeholder_shape,
name='positive_input')
negative_input = tf.placeholder(tf.float32,
placeholder_shape,
name='negative_input')
def train():
data_src = './ChallengeImages2/'
learning_rate = 0.01
train_iter = 250
batch_size = 128
momentum = 0.99
step = 50
# Setup Dataset
dataset = PreProcessing(data_src)
print(dataset.images_test)
print(dataset.labels_test)
model = TripletLoss()
height = 50
width = 150
dims = [height, width, 3]
placeholder_shape = [None] + dims
print("placeholder_shape", placeholder_shape)
# Setup Network
next_batch = dataset.get_triplets_batch
anchor_input = tf.placeholder(tf.float32,
placeholder_shape,
name='anchor_input')
positive_input = tf.placeholder(tf.float32,
placeholder_shape,
name='positive_input')
negative_input = tf.placeholder(tf.float32,
placeholder_shape,
name='negative_input')
margin = 0.5
# Will be of size N x 28
anchor_output = model.conv_net(anchor_input, reuse=tf.AUTO_REUSE)
positive_output = model.conv_net(positive_input, reuse=tf.AUTO_REUSE)
negative_output = model.conv_net(negative_input, reuse=tf.AUTO_REUSE)
print(tf.shape(anchor_output))
return
'''
compute the similarity between the positive_output and the negative_output
if similarity is < 0.25 that's great
'''
similarity_pos_neg = computeCosSimilarity(positive_output, negative_output)
similarity_pos_anchor = computeCosSimilarity(positive_output,
anchor_output)
loss = model.triplet_loss(anchor_output, positive_output, negative_output,
margin)
# Setup Optimizer
global_step = tf.Variable(0, trainable=False)
train_step = tf.train.MomentumOptimizer(learning_rate,
momentum,
use_nesterov=True).minimize(
loss, global_step=global_step)
# Start Training
saver = tf.train.Saver()
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
# Setup Tensorboard
tf.summary.scalar('step', global_step)
tf.summary.scalar('loss', loss)
for var in tf.trainable_variables():
tf.summary.histogram(var.op.name, var)
merged = tf.summary.merge_all()
writer = tf.summary.FileWriter('train.log', sess.graph)
# Train iter
for i in range(train_iter):
batch_anchor, batch_positive, batch_negative = next_batch(
batch_size)
_, l, summary_str = sess.run(
[train_step, loss, merged],
feed_dict={
anchor_input: batch_anchor,
positive_input: batch_positive,
negative_input: batch_negative
})
#pNAccuracy = computeAccuracyLess(0.25, similarity_p_n)
#pAAccuracy = computeAccuracyGreat(0.75, similarity_p_a)
writer.add_summary(summary_str, i)
print("#%d - Loss" % i, l)
#print("#%d - P and A Accuracy" % i, pAAccuracy)
#print("#%d - N and A Accuracy" % i, pNAccuracy)
if (i + 1) % step == 0:
saver.save(sess, "model_triplet/model.ckpt")
saver.save(sess, "model_triplet/model.ckpt")
print('Training completed successfully.')
return dataset
Y = np.asarray([label_dict[label] for label in y])
shuffle_indices = np.random.permutation(np.arange(len(y)))
x_shuffled = []
y_shuffled = []
for index in shuffle_indices:
x_shuffled.append(X[index])
y_shuffled.append(Y[index])
size_of_dataset = len(x_shuffled)
n_train = int(np.ceil(size_of_dataset * train_test_ratio))
n_test = int(np.ceil(size_of_dataset * (1 - train_test_ratio)))
return np.asarray(x_shuffled[0:n_train]), np.asarray(x_shuffled[n_train + 1:size_of_dataset]), np.asarray(y_shuffled[0:n_train]), np.asarray(y_shuffled[n_train + 1:size_of_dataset])
train_images, test_images, train_label, test_label = get_train_test_dataset(0.7)
model_path = './model_triplet/'
model = TripletLoss()
# Input and output tensor
img_placeholder = tf.placeholder(tf.float32, [None, 28, 28, 3], name='img')
net = model.conv_net(img_placeholder, reuse=False)
print(net)
# generate random index from test image corpus and display the image
idx = np.random.randint(0, len(test_images))
im = test_images[idx]
# show the test image
print('************Query Image**************')
#show_image(idx, test_images)
# Find k-nearest neighbor using cosine similarity
tf.flags.DEFINE_integer('max_document_length',15,'maximum number of word in a sentence')
tf.flags.DEFINE_float('momentum','0.99', 'Momentum')
tf.flags.DEFINE_string('model', 'conv_net', 'model to run')
tf.flags.DEFINE_string('data_src', './data_repository/questions.csv', 'source of training dataset')
flags = tf.app.flags
FLAGS = flags.FLAGS
if __name__ == "__main__":
# Setup Dataset
dataset = PreProcessing(FLAGS.data_src, FLAGS.max_document_length)
model = TripletLoss(sequence_length=dataset.X.shape[1],
vocab_size=len(dataset.vocab_processor.vocabulary_),
embedding_size=FLAGS.embedding_dim,
filter_sizes=list(map(int, FLAGS.filter_sizes.split(","))),
num_filters=FLAGS.num_filters,
output_embedding_size = FLAGS.output_embedding_size,
dropout_keep_prob = FLAGS.dropout_keep_prob,
embeddings_lookup= dataset.embeddings_lookup,
l2_reg_lambda=FLAGS.l2_reg_lambda)
placeholder_shape = [None] + [dataset.X.shape[1]]
print("placeholder_shape", placeholder_shape)
# Setup Network
next_batch = dataset.get_triplets_batch
anchor_input = tf.placeholder(tf.int32, placeholder_shape, name='anchor_input')
positive_input = tf.placeholder(tf.int32, placeholder_shape, name='positive_input')
negative_input = tf.placeholder(tf.int32, placeholder_shape, name='negative_input')
margin = 3.5
anchor_output = model.conv_net(anchor_input, reuse=False)