def train():
with graph.as_default():
mac_remove_file()
starttime=time.time()
datamodel = Dataset(False)
print("Done with the file-creation")
datamodel.make_path_lists()
print("Done with the path-creation")
print("Time for path-creation is:",time.time() - starttime)
nb_categories=datamodel.class_num
print(nb_categories)
tb = TensorBoard(log_dir=os.path.join('logs'))
checkpoint= ModelCheckpoint(filepath=os.path.join('checkpoints','lstm-'+'.{epoch:03d}-{val_loss:.3f}.hdf5'),verbose=1,save_best_only=True)
early_stopper = EarlyStopping(patience=15)
feature_dim= 2048
sampling_rate=40
batchsize=6
epochs= 100
steps_per_epoch= datamodel.trainlength//batchsize
steps_per_epoch_test = datamodel.testlength//batchsize
dnn_model = lstm_model(nb_categories,sampling_rate,feature_dim)
model= dnn_model.getmodel()
# train_generator= datamodel.train_data_generator(batchsize)
# test_generator= datamodel.test_data_generator( batchsize)
starttime=time.time()
# model.fit_generator(generator=train_generator,steps_per_epoch=steps_per_epoch,epochs=epochs,verbose=1,validation_data=test_generator,validation_steps=steps_per_epoch_test,callbacks=[tb])
X,y = datamodel.load_all_in_memory(datamodel.trainlist)
X_test,y_test = datamodel.load_all_in_memory(datamodel.testlist)
print("Time for loading into memory is:",time.time() - starttime)
starttime=time.time()
model.fit(X,y,validation_data=(X_test, y_test),batch_size=batchsize,verbose=1,epochs=epochs,callbacks=[tb,checkpoint,early_stopper])
print("Training-Validation time is",time.time() - starttime)
def test(sess):
test_model = lstm_model(False, 1, 1)
test_model.restore(sess)
test_batches = make_batches(FLAGS.test_data, 1, 1)
_, test_perplexity, __, ___ = run_epoch(
sess,
test_model,
test_batches,
tf.no_op(),
0,
0,
)
print('finished test, test perplexity: {}'.format(test_perplexity))
def stock_predict(self):
print(self.ticker)
print("self.ticker:", self.ticker)
l = lstm_model()
return l.execute(self.ticker)
# def main():
# st = Stock("#MSFT")
# # print(st.daily_stock_data("GOOG"))
# print("Predicted:",st.stock_predict())
#
# if __name__=="__main__":
# main()
def __init__(self, checkpoint):
"""Constructor. Load vocabulary index, start token, end token, unk id,
mask_id. Restore checkpoint.
Args:
checkpoint: A path to the checkpoint
"""
self.word_to_id, self.idx_to_word, self.embedding, self.start_id, self.end_id, self.unk_id, self.mask_id = load_sentence_embeddings()
self.checkpoint = checkpoint
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.5)
self.sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))
self.model = lstm_model(self.sess, 'infer', 300, self.embedding, self.start_id, self.end_id, self.mask_id)
saver = tf.train.Saver()
saver.restore(self.sess, checkpoint)
def train():
with graph.as_default():
datamodel = Dataset()
datamodel.make_path_lists()
nb_categories=datamodel.class_num
print(nb_categories)
tb = TensorBoard(log_dir=os.path.join('logs'))
feature_dim= 2048
sampling_rate=40
batchsize=6
epochs= 2
steps_per_epoch= datamodel.trainlength//batchsize
steps_per_epoch_test = datamodel.testlength//batchsize
dnn_model = lstm_model(nb_categories,sampling_rate,feature_dim)
model= dnn_model.getmodel()
train_generator= datamodel.train_data_generator(batchsize)
test_generator= datamodel.test_data_generator( batchsize)
model.fit_generator(generator=train_generator,steps_per_epoch=steps_per_epoch,epochs=2,verbose=1,validation_data=test_generator,validation_steps=steps_per_epoch_test,callbacks=[tb])
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--num_lstm_layers', type=int, default=2,
help='num_lstm_layers')
parser.add_argument('--fc7_feature_length', type=int, default=4096,
help='fc7_feature_length')
parser.add_argument('--rnn_size', type=int, default=128,
help='rnn_size')
parser.add_argument('--embedding_size', type=int, default=128,
help='embedding_size'),
parser.add_argument('--word_emb_dropout', type=float, default=0.5,
help='word_emb_dropout')
parser.add_argument('--image_dropout', type=float, default=0.5,
help='image_dropout')
parser.add_argument('--data_dir', type=str, default='data',
help='Data directory')
parser.add_argument('--batch_size', type=int, default=32,
help='Batch Size')
parser.add_argument('--learning_rate', type=float, default=0.001,
help='Batch Size')
parser.add_argument('--epochs', type=int, default=20,
help='Expochs')
parser.add_argument('--debug', type=bool, default=False,
help='Debug')
parser.add_argument('--resume_model', type=str, default=None,
help='Trained Model Path')
parser.add_argument('--version', type=int, default=2,
help='VQA data version')
parser.add_argument('--model', type=str, default="cnn_lstm_model", help='model to run')
args = parser.parse_args()
print("Reading QA DATA")
qa_data = data_loader.load_data(args.data_dir)
print("Reading fc7 features")
fc7_features, image_id_list = data_loader.load_fc7_features(args.data_dir, 'train')
print("FC7 features", fc7_features.shape)
print("image_id_list", image_id_list.shape)
# map and image to an id
image_id_map = {}
for i in range(len(image_id_list)):
image_id_map[image_id_list[i]] = i
# id to word
ans_map = {qa_data['topic_vocab'][ans]: ans for ans in qa_data['topic_vocab']}
lstm_steps = qa_data['max_conversation_length'] if args.model == "lstm_model" else qa_data[
'max_conversation_length'] + 1
model_options = {
'num_lstm_layers': args.num_lstm_layers,
'rnn_size': args.rnn_size,
'embedding_size': args.embedding_size,
'word_emb_dropout': args.word_emb_dropout,
'image_dropout': args.image_dropout,
'fc7_feature_length': args.fc7_feature_length,
'lstm_steps': lstm_steps,
'q_vocab_size': len(qa_data['conversation_vocab']),
'ans_vocab_size': len(qa_data['topic_vocab'])
}
if args.model == "cnn_lstm_model":
model = lstm_model.lstm_model(model_options)
elif args.model == "cnn_model":
model = cnn_model.cnn_model(model_options)
elif args.model == "lstm_model":
model = lstm_model.lstm_model(model_options)
else:
model = None
input_tensors, t_loss, t_accuracy, t_p = model.build_model()
train_op = tf.train.AdamOptimizer(args.learning_rate).minimize(t_loss)
sess = tf.InteractiveSession()
tf.initialize_all_variables().run()
saver = tf.train.Saver()
if args.resume_model:
saver.restore(sess, args.resume_model)
for i in range(args.epochs):
batch_no = 0
while (batch_no * args.batch_size) < len(qa_data['training']):
sentence, answer, fc7 = get_training_batch(batch_no, args.batch_size, fc7_features, image_id_map, qa_data,
'train')
_, loss_value, accuracy, pred = sess.run([train_op, t_loss, t_accuracy, t_p],
feed_dict={
input_tensors['fc7']: fc7,
input_tensors['sentence']: sentence,
input_tensors['answer']: answer
}
)
batch_no += 1
if args.debug:
for idx, p in enumerate(pred):
print(ans_map[p], ans_map[np.argmax(answer[idx])])
print("Loss", loss_value, batch_no, i)
print("Accuracy", accuracy)
print("---------------")
else:
print("Loss", loss_value, batch_no, i)
print("Training Accuracy", accuracy)
saver.save(sess, "data/models/{}{}.ckpt".format(args.model, i))
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--num_lstm_layers',
type=int,
default=2,
help='num_lstm_layers')
parser.add_argument('--fc7_feature_length',
type=int,
default=4096,
help='fc7_feature_length')
parser.add_argument('--rnn_size', type=int, default=128, help='rnn_size')
parser.add_argument('--embedding_size',
type=int,
default=128,
help='embedding_size'),
parser.add_argument('--word_emb_dropout',
type=float,
default=0.5,
help='word_emb_dropout')
parser.add_argument('--image_dropout',
type=float,
default=0.5,
help='image_dropout')
parser.add_argument('--data_dir',
type=str,
default='data',
help='Data directory')
parser.add_argument('--batch_size',
type=int,
default=10,
help='Batch Size')
# parser.add_argument('--learning_rate', type=float, default=0.001,
# help='Batch Size')
# parser.add_argument('--epochs', type=int, default=20,
# help='Expochs')
parser.add_argument('--debug', type=bool, default=False, help='Debug')
parser.add_argument('--model_path',
type=str,
default='19',
help='Model Path')
parser.add_argument('--version',
type=int,
default=2,
help='VQA data version')
parser.add_argument('--model',
type=str,
default="cnn_lstm_model",
help='model to run')
args = parser.parse_args()
print("Reading QA DATA")
qa_data = data_loader.load_data(args.data_dir)
print("Reading fc7 features")
fc7_features, image_id_list = data_loader.load_fc7_features(
args.data_dir, 'val')
print("FC7 features", fc7_features.shape)
print("image_id_list", image_id_list.shape)
image_id_map = {}
for i in range(len(image_id_list)):
image_id_map[image_id_list[i]] = i
ans_map = {
qa_data['topic_vocab'][ans]: ans
for ans in qa_data['topic_vocab']
}
lstm_steps = qa_data[
'max_conversation_length'] if args.model == "lstm_model" else qa_data[
'max_conversation_length'] + 1
model_options = {
'num_lstm_layers': args.num_lstm_layers,
'rnn_size': args.rnn_size,
'embedding_size': args.embedding_size,
'word_emb_dropout': args.word_emb_dropout,
'image_dropout': args.image_dropout,
'fc7_feature_length': args.fc7_feature_length,
'lstm_steps': lstm_steps,
'q_vocab_size': len(qa_data['conversation_vocab']),
'ans_vocab_size': len(qa_data['topic_vocab'])
}
if args.model == "cnn_lstm_model":
model = lstm_model.lstm_model(model_options)
elif args.model == "cnn_model":
model = cnn_model.cnn_model(model_options)
elif args.model == "lstm_model":
model = lstm_model.lstm_model(model_options)
else:
model = None
input_tensors, t_prediction, t_ans_probab = model.build_generator()
sess = tf.InteractiveSession()
saver = tf.train.Saver()
avg_accuracy = 0.0
total = 0
model_path = "data/models/" + args.model + args.model_path + ".ckpt"
saver.restore(sess, model_path)
batch_no = 0
while (batch_no * args.batch_size) < len(qa_data['validation']):
sentence, answer, fc7 = get_batch(batch_no, args.batch_size,
fc7_features, image_id_map, qa_data,
'val')
pred, ans_prob = sess.run([t_prediction, t_ans_probab],
feed_dict={
input_tensors['fc7']: fc7,
input_tensors['sentence']: sentence,
})
batch_no += 1
if args.debug:
for idx, p in enumerate(pred):
print(ans_map[p], ans_map[np.argmax(answer[idx])])
correct_predictions = np.equal(pred, np.argmax(answer, 1))
correct_predictions = correct_predictions.astype('float32')
accuracy = correct_predictions.mean()
print("Acc", accuracy)
avg_accuracy += accuracy
total += 1
print("Acc", avg_accuracy / total)
import sys
import subprocess
import os
import numpy as np
from keras.callbacks import TensorBoard, ModelCheckpoint, EarlyStopping, CSVLogger
from create_images import extract_images, sample_x_images, destination_directory, remove_and_create
from lstm_model import lstm_model
from datagen import Dataset
nb_categories = 5
feature_dim = 2048
sampling_rate = 40
dest_dir = "Validation_Data"
dnn_model = lstm_model(nb_categories, sampling_rate, feature_dim)
model = dnn_model.getmodel()
model.load_weights('/Users/s0j00os/Documents/Proj/lstmodel.hdf5')
FNULL = open(os.devnull, 'w')
datamodel = Dataset()
videoname = sys.argv[1]
remove_and_create(dest_dir)
subprocess.call(
['ffmpeg', '-i', videoname,
os.path.join(dest_dir, 'image-%04d.jpg')],
stdout=FNULL,
stderr=subprocess.STDOUT)
imagelist = sample_x_images(dest_dir, sampling_rate)
cnn_sequence = datamodel.build_sequence(imagelist)
y = model.predict_classes(np.array([np.array(cnn_sequence)]))
print("The prediction is :", datamodel.class_dict[y[0]])
X_train, X_val, y_train, y_val = train_test_split(X_train,
y_train,
test_size=0.25,
random_state=42)
# Embedding Stuff
vocab = vectorizer.get_vocabulary()
word_index = dict(zip(vocab, range(2, len(vocab))))
num_tokens = len(vocab) + 2
embedding_dim = 300
embedding_matrix = embed_matrix_gen(word_index, num_tokens, embedding_dim)
conv_model = conv_model(embedding_matrix, MAX_SEQ_LEN, num_tokens,
embedding_dim)
lstm_model = lstm_model(embedding_matrix, MAX_SEQ_LEN, num_tokens,
embedding_dim)
tf_model = tf_model(MAX_SEQ_LEN, num_tokens)
# models = [conv_model, lstm_model, tf_model]
for model_str in ['tf_model', 'conv_model', 'lstm_model']:
model = eval(model_str)
model_callbacks = [
ks.callbacks.EarlyStopping(monitor='val_acc',
patience=3,
restore_best_weights=True)
]
if model_str != 'tf_model':
model_callbacks.append(
ks.callbacks.ModelCheckpoint(filepath=f'models/{model_str}.h5',
monitor='val_acc',
save_best_only=True))
model.summary()
# Variable to save the number of steps in the sequence
TIME_STEPS = 128
# Step to recolect samples every STEP times a new sample of TIME_STEP will be recolected
STEP = 40
# Units in NN
UNITS = 128
# No. of classes for classification
CLASSES = 2
# Batch size
BATCH_SIZE = 64
# Learning rate
LEARNING_RATE = 0.0001
# Adam
BETA_1 = 0.9
BETA_2 = 0.999
X_train, y_train, X_test, y_test = load_data(TIME_STEPS, STEP)
har_lst = lstm_model(X_train, y_train, CLASSES, UNITS, BATCH_SIZE,
LEARNING_RATE, BETA_1, BETA_2)
y_true, y_predicted = har_lst.train(100)
plot_cm(y_true, y_predicted, [0, 1])
har_lst.predict(X_test, y_test)
def train(sess):
with tf.name_scope('Train'):
with tf.variable_scope('language_model', reuse=None):
train_model = lstm_model(True, FLAGS.batch_size, FLAGS.num_steps)
train_model.set_saver('lstm-%d-%d' %
(FLAGS.num_epoches, FLAGS.word_dim))
with tf.name_scope('Test'):
with tf.variable_scope('language_model', reuse=True):
test_model = lstm_model(False, 1, 1)
tf.global_variables_initializer().run()
train_batches = make_batches(FLAGS.train_data, FLAGS.batch_size,
FLAGS.num_steps)
valid_batches = make_batches(FLAGS.valid_data, 1, 1)
epoch = 1
n = 1
best_step = n
best = .0
start = time.time()
origin_begin = start
while epoch <= FLAGS.num_epoches:
total_losses = 0.0
iters = 0
train_state = sess.run(train_model.initial_state)
for x, y in train_batches:
fetches = [
train_model.losses, train_model.final_state,
train_model.train_op
]
loss, state, _ = sess.run(
fetches, {
train_model.input_data: x,
train_model.label: y,
train_model.initial_state: train_state
})
total_losses += loss
iters += train_model.num_steps
valid_perp = 0
test_state = sess.run(test_model.initial_state)
for test_x, test_y in valid_batches:
valid_perp += sess.run(
test_model.perp, {
test_model.input_data: test_x,
test_model.label: test_y,
test_model.initial_state: test_state
})
valid_perp = valid_perp / len(valid_batches)
perp = np.exp(total_losses / iters)
if best < valid_perp:
best = valid_perp
best_step = n
end = time.time()
print('----epoch: {} perp: {} best_perp: {} time: {} ----'.format(
epoch, perp, valid_perp, end - start))
start = end
epoch += 1
duration = (time.time() - origin_begin) / 3600
train_model.save(sess, best_step)
print('Done training! best_step: {}, best_perp: {}'.format(
best_step, best))
print('duration: %.2f hours' % duration)