def gen_test():
gen = generator.generator(10)
X,y = next(gen.generate_training_data())
#Y = np.array(X)
#print("Y.dimensions(): ", np.shape(X))
print(X.shape)
print(y.shape)
def main(_):
# Create global_step
filenames, class_names = generator.get_filenames_and_classes(
FLAGS.data_dir)
random.shuffle(filenames)
training_filenames = filenames[val_num:]
validation_filenames = filenames[:val_num]
train_data_len = len(training_filenames)
val_data_len = len(validation_filenames)
global_step = tf.contrib.framework.get_or_create_global_step()
#create placeholder for image, labels, is_training
images = tf.placeholder(tf.float32,
shape=(None, FLAGS.image_size, FLAGS.image_size,
3))
labels = tf.placeholder(tf.int32, shape=(None, ))
is_training_tensor = tf.placeholder(tf.bool)
#create vgg_19 model
arg_scope = vgg_arg_scope()
with slim.arg_scope(arg_scope):
logits, end_points = vgg_19(images,
num_classes=FLAGS.class_num,
is_training=is_training_tensor)
#create loss tensor
one_hot_labels = tf.one_hot(labels, FLAGS.class_num)
total_loss = losses(logits, labels)
#create acc tensor
predictions = tf.argmax(logits, 1)
ground_truth = tf.argmax(one_hot_labels, 1)
acc = tf.reduce_mean(tf.cast(tf.equal(predictions, ground_truth), "float"))
tf.summary.scalar('acc', acc)
#optimizer setting
optimizer = tf.train.AdamOptimizer(learning_rate=1e-3)
train_op = optimizer.minimize(total_loss, global_step=global_step)
#summary merge
merged = tf.summary.merge_all()
#create log dir and file write
date_time_str = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
log_dir = FLAGS.log_dir + "/" + date_time_str
file_writer = tf.summary.FileWriter(log_dir, tf.get_default_graph())
#cretae summary, saver
duration_summary = tf.Summary()
saver = tf.train.Saver()
init_op = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
with tf.Session() as sess:
sess.run(init_op)
# create data generator
train_gen = generator.generator(training_filenames,
class_names,
train_data_len,
FLAGS.batch_size,
FLAGS.epoch_num,
sess,
FLAGS.image_size,
FLAGS.image_size,
is_training=True)
val_gen = generator.generator(validation_filenames,
class_names,
val_data_len,
FLAGS.batch_size,
FLAGS.epoch_num,
sess,
FLAGS.image_size,
FLAGS.image_size,
is_training=False)
#calc train_step_num with batch size and epoch num
train_step_num = int(
np.ceil((train_num * FLAGS.epoch_num) / FLAGS.batch_size))
#calc 1 epoch val_step_num with batch size
val_step_num = int(np.ceil(val_num / FLAGS.batch_size))
#calc 1 epoch train step num
epoch_step_num = int(np.ceil(train_num / FLAGS.batch_size))
epoch_num = 0
train_loss_list = []
train_acc_list = []
step = 0
print("step num", train_step_num, val_step_num, epoch_step_num)
for i in range(train_step_num):
start_time = time.time()
train_images_batch, train_labels_batch = next(train_gen)
_, loss_train, acc_train = sess.run(
[train_op, total_loss, acc],
feed_dict={
images: train_images_batch,
labels: train_labels_batch,
is_training_tensor: True
})
train_loss_list.append(loss_train)
train_acc_list.append(acc_train)
duration = time.time() - start_time
step += 1
print("train step", step, "shape", np.shape(train_images_batch),
np.shape(train_labels_batch), "duration ", duration)
# each epoch calculate validate loss print loss
if (step + 1) % epoch_step_num == 0:
avg_train_loss = np.mean(train_loss_list)
avg_train_acc = np.mean(train_acc_list)
train_loss_list = []
train_acc_list = []
avg_val_loss, avg_val_acc = validate_run(
sess, images, labels, is_training_tensor, val_step_num,
val_gen, total_loss, acc)
print("epoch: ", epoch_num, "train:", avg_train_loss,
avg_train_acc, "validate: ", avg_val_loss, avg_val_acc)
epoch_num += 1
saver.save(sess, FLAGS.ckpt_name)
# ecah 10 step save log
if step % 10 == 0:
summary = sess.run(merged,
feed_dict={
images: train_images_batch,
labels: train_labels_batch,
is_training_tensor: True
})
duration_summary.value.add(tag="step_duration",
simple_value=duration)
file_writer.add_summary(duration_summary, step)
file_writer.add_summary(summary, step)
print("last step:", step, train_step_num, FLAGS.epoch_num)
file_writer.close()
import os
import numpy as np #linear algebra
import pandas as pd #data preprocessing
#import matplotlib.pyplot as plt #data visualization
import h5py
import PIL
import tensorflow as tf
import data.generator as generator #geneartes a single batch of data
gen = generator.generator(4)
X = tf.placeholder(tf.float32, shape=([1, 500,500,3]))
with tf.Session() as sess:
model = tf.keras.models.load_model('../models/model_ckpt_4.h5')
print("testing on a single batch... ")
#print a single prediction as well as the expected prediction
#X = tf.placeholder(tf.float32, shape=([1, 500,500,3]))
X,y = next(gen.generate_testing_data())
#X = [X[0],X[1]]
#X = np.vstack(X)
#X = np.expand_dims(X, axis=0)
classification = model.predict(X, batch_size=4)
print("classifications: ",classification)
print("Actual Classification part 1: ", y[0])
print("\nActual Classficiation part 2: ", y[1])
def __init__(self):
self.train = False
self.test = True
self.load_model = True
#basic parameters
self.image_size = 224
self.batch_size = 128
self.num_classes = 120
self.i = 6
#self.is_training = tf.placeholder(tf.bool)
self.EPOCHS = 25
self.INITIALIZATION_EPOCHS = 3
#set the input and output placeholders
#self.X = tf.placeholder(tf.float32, shape=([None, 500,500,3]))
#self.y = tf.placeholder(tf.float32, shape=([None, 120, 1]))
self.X = tf.keras.layers.Input(shape=(self.image_size, self.image_size,
3),
batch_size=self.batch_size,
name='input_data',
dtype='float32')
self.y = tf.placeholder(tf.float32,
shape=[None, 120, 1],
name='correct_labels')
self.y_pred = tf.placeholder(tf.float32,
shape=[None, 120, 1],
name='predicted_labels')
#get pretrained neural network
self.base_model = self.get_inception_resnet_v2()
#update the input and output layers of the model
self.model = tf.keras.Model(inputs=self.base_model.input,
outputs=self.generate_output_layer())
#access the generator for use during any training/testing sess
self.gen = generator.generator(self.batch_size)
if (self.i == 0 and self.train
== True): #if on the first iteration of training
print("Initial layers of inception resnet v2 will now be trained")
self.initial_train(
) #train the last nodes on the new output classes
if (self.train):
print("Program is now in training mode")
self.train_model() #train the entire model
#evaulate the model
if (self.test):
self.evaluate_model()
#test a single image
#self.use_model()
#save the model
#self.save_model()
quit()
def train():
# hyperparameters + files
DATA_DIR = 'data/'
IMAGE_DIR = DATA_DIR + 'images/'
DATASET = 'dataset.csv'
MODEL_DIR = DATA_DIR + 'saved_model'
VOCAB = 'vocab.txt'
BATCH_SIZE = 16
EPOCHS = 10
START_EPOCH = 0
IMAGE_DIM = (128, 416)
load_saved_model = False
max_equation_length = 200 + 2
vocabFile = open(DATA_DIR + VOCAB, 'r', encoding="utf8")
vocab_tokens = [x.replace('\n', '') for x in vocabFile.readlines()]
vocabFile.close()
vocab_size = len(vocab_tokens)
# import the equations + image names and the tokens
dataset = pd.read_csv(DATA_DIR + DATASET)
dataset['Y'] = dataset['latex_equations'].apply(
lambda x: encode_equation(x, vocab_tokens, max_equation_length, False))
dataset['Y_loss'] = dataset['latex_equations'].apply(
lambda x: encode_equation(x, vocab_tokens, max_equation_length, True))
train_idx, val_idx = train_test_split(dataset.index,
random_state=92372,
test_size=0.20)
# the validation and training data generators
train_generator = generator(list_IDs=train_idx,
df=dataset,
base_path=IMAGE_DIR,
shuffle=True)
val_generator = generator(list_IDs=val_idx,
df=dataset,
base_path=IMAGE_DIR,
shuffle=True)
train_generator = DataLoader(train_generator,
batch_size=BATCH_SIZE,
shuffle=True,
num_workers=2,
pin_memory=True)
val_generator = DataLoader(val_generator,
batch_size=BATCH_SIZE,
shuffle=True,
num_workers=2,
pin_memory=True)
# initialize our model
model = im2latex(vocab_size)
optimizer = optim.Adam(model.parameters(), lr=1e-3)
lr_schedule = optim.lr_scheduler.ReduceLROnPlateau(optimizer,
factor=0.95,
patience=1,
verbose=True,
cooldown=1,
min_lr=1e-7)
epsilon = 1.0
best_val_loss = np.inf
if load_saved_model:
checkpoint = torch.load(MODEL_DIR + '/best_ckpt.pt')
model.load_state_dict(checkpoint['model_state_dict'])
model.cuda()
optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
START_EPOCH = checkpoint['epoch']
best_val_loss = checkpoint['best_val_loss']
#epsilon = checkpoint['epsilon']
print('Loaded weights')
print(f'epsilon val: {epsilon}')
print(f'best_val_loss: {best_val_loss}')
print(f'model summary: {model}')
input()
trainer = Trainer(optimizer=optimizer,
loss_fn=loss_fn,
model=model,
train_generator=train_generator,
val_generator=val_generator,
model_path=MODEL_DIR,
lr_scheduler=lr_schedule,
init_epoch=START_EPOCH,
epsilon=epsilon,
best_val_loss=best_val_loss,
num_epochs=EPOCHS)
trainer.train()
# import the equations + image names and the tokens
dataset = pd.read_csv(DATA_DIR+DATASET)
vocabFile = open(DATA_DIR+VOCAB, 'r', encoding="utf8")
vocab_tokens = [x.replace('\n', '') for x in vocabFile.readlines()]
vocabFile.close()
vocab_size = len(vocab_tokens)
dataset['Y'] = dataset['latex_equations'].apply(lambda x: encode_equation(x, vocab_tokens, max_equation_length, False))
dataset['Y_loss'] = dataset['latex_equations'].apply(lambda x: encode_equation(x, vocab_tokens, max_equation_length, True))
train_idx, val_idx = train_test_split(
dataset.index, random_state=92372, test_size=0.20
)
# the validation and training data generators
train_generator = generator(list_IDs=train_idx,
df=dataset,
base_path=IMAGE_DIR,
shuffle=True)
val_generator = generator(list_IDs=val_idx,
df=dataset,
base_path=IMAGE_DIR,
shuffle=True)
# the model
model = im2latex(vocab_size)
epsilon = 1.0
if load_saved_model:
print('Loading weights')
checkpoint = torch.load(MODEL_DIR + 'ckpt-33-0.9793.pt')
model.load_state_dict(checkpoint['model_state_dict'])
epsilon = checkpoint['epsilon']