def run_experiment(experiment_type, data_folder, save_model_folder,
save_results_folder):
"""
Runs experiments and saves results
Parameters
----------
experiment_type
data_folder
save_model_folder
save_results_folder
"""
def set_experiment_variables(hidden_state_size=512,
down_project_size=None,
load_embeddings=False):
tf.flags.DEFINE_integer("hidden_state_size", hidden_state_size,
"hidden state size (default 512)")
tf.flags.DEFINE_integer(
"down_project_size", down_project_size,
"Down projection size. Should be used with a hidden_state_size of 1024 (default None)"
)
tf.flags.DEFINE_boolean(
"load_embeddings", load_embeddings,
"Whether to use pretrained embeddings or not (default False)")
if experiment_type == 'A':
set_experiment_variables(512, None, False)
elif experiment_type == 'B':
set_experiment_variables(512, None, True)
elif experiment_type == 'C':
set_experiment_variables(1024, 512, True)
print("\nExperiment Arguments:")
for key in FLAGS.flag_values_dict():
if key == 'f':
continue
print("{:<22}: {}".format(key.upper(), FLAGS[key].value))
print(" ")
data_processing = DataProcessing(FLAGS.sentence_length,
FLAGS.max_vocabulary_size)
train_corpus = data_processing.preprocess_dataset(data_folder,
'sentences.train')
validation_corpus = data_processing.preprocess_dataset(
data_folder, 'sentences.eval')
test_corpus = data_processing.preprocess_dataset(data_folder,
'sentences_test.txt')
continuation_corpus = data_processing.preprocess_dataset(
data_folder, 'sentences.continuation', pad_to_sentence_length=False)
print(f'Number of train sentences is \t\t{len(train_corpus)}')
print(f'Number of validation sentences is \t{len(validation_corpus)}')
print(f'Number of test sentences is \t\t{len(test_corpus)}')
print(f'Number of continuation sentences is \t{len(continuation_corpus)}')
print(" ")
best_perplexity = None
best_model = None
with tf.Graph().as_default():
with tf.Session() as session:
# Create a variable to contain a counter for the global training step.
global_step = tf.Variable(1, name='global_step', trainable=False)
lstm = LSTMCell(FLAGS.embedding_size,
FLAGS.hidden_state_size,
FLAGS.sentence_length,
FLAGS.max_vocabulary_size,
down_project_size=FLAGS.down_project_size,
pad_symbol=data_processing.vocab['<pad>'])
if FLAGS.load_embeddings:
load_embedding(session, data_processing.vocab,
lstm.input_embeddings,
data_folder + '/wordembeddings-dim100.word2vec',
FLAGS.embedding_size,
len(data_processing.vocab))
####
# Set optimizer and crop all gradients to values [-5, 5]
####
with tf.name_scope('train'):
optimizer = tf.train.AdamOptimizer()
gvs = optimizer.compute_gradients(lstm.loss)
capped_gvs = [(tf.clip_by_value(grad, -5., 5.), var)
for grad, var in gvs]
train_step = optimizer.apply_gradients(capped_gvs,
global_step=global_step)
saver = tf.train.Saver(tf.global_variables(),
max_to_keep=FLAGS.num_checkpoints)
session.run(tf.global_variables_initializer())
summaries_merged = tf.summary.merge(lstm.summaries)
####
# Create checkpoint directory
####
timestamp = str(int(time.time()))
out_dir = os.path.abspath(
os.path.join(save_model_folder, "runs", timestamp))
checkpoint_dir = os.path.abspath(
os.path.join(out_dir, "checkpoints"))
if not os.path.exists(checkpoint_dir):
os.makedirs(checkpoint_dir)
####
# Start training for the specified epochs
####
print('Start training...')
for epoch in range(FLAGS.num_epochs):
for sentences_batch in get_batches(
train_corpus, batch_size=FLAGS.batch_size):
# run a single step
train_batch(sentences_batch, lstm, train_step, global_step,
session, summaries_merged)
current_step = tf.train.global_step(session, global_step)
if current_step % FLAGS.checkpoint_every == 0:
perplexities = dev_step(
get_batches(validation_corpus,
batch_size=FLAGS.batch_size,
do_shuffle=False), lstm, global_step,
session)
average_perplexity = np.mean(perplexities)
model_name = "model_experiment-{}_epoch-{}_val-perplexity-{}".format(
experiment_type, epoch + 1, average_perplexity)
path = saver.save(session,
os.path.join(checkpoint_dir, model_name))
print("Saved model checkpoint to {}".format(path))
if best_perplexity is None or best_perplexity > average_perplexity:
best_perplexity = average_perplexity
best_model = model_name
print('Done with epoch', epoch + 1)
if best_model is None:
raise Exception(
"Model has not been saved. Run for at least one epoch")
print('Restoring best model', best_model)
saver.restore(session, os.path.join(checkpoint_dir, best_model))
# evaluate on test set
perplexities = dev_step(get_batches(test_corpus,
batch_size=FLAGS.batch_size,
do_shuffle=False),
lstm,
global_step,
session,
verbose=0)
print('Perplexity on test_set is', np.mean(perplexities))
filename = "group25.perplexity{}".format(experiment_type)
savefile = os.path.join(save_results_folder, filename)
print('Saving results to', savefile)
with open(savefile, 'w') as f:
f.writelines(str(i) + '\n' for i in perplexities)
if experiment_type == 'C':
continuation_sentences = continue_sentences(
continuation_corpus, session, lstm, data_processing)
filename = "group25.continuation"
savefile = os.path.join(save_results_folder, filename)
print('Saving results to', savefile)
with open(savefile, 'w') as f:
f.writelines(str(i) + '\n' for i in continuation_sentences)
print('Done')
saver = tf.train.Saver()
summary_op = tf.summary.merge_all()
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
train_summary_writer = tf.summary.FileWriter(TRAIN_LOG_DIR, sess.graph)
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
step = 0
for epoch in range(EPOCH_NUM):
accuracy_epoch = 0
loss_epoch = 0
batch_index = 0
for i in range(len(train_video_indices) // BATCH_SIZE):
step += 1
batch_data, batch_index = data_processing.get_batches(TRAIN_LIST_PATH, NUM_CLASSES, batch_index,
train_video_indices, BATCH_SIZE)
_, loss_out, accuracy_out, summary = sess.run([optimizer, loss, accuracy, summary_op],
feed_dict={batch_clips:batch_data['clips'],
batch_labels:batch_data['labels'],
keep_prob: 0.5})
loss_epoch += loss_out
accuracy_epoch += accuracy_out
if i % 10 == 0:
print('Epoch %d, Batch %d: Loss is %.5f; Accuracy is %.5f'%(epoch+1, i, loss_out, accuracy_out))
train_summary_writer.add_summary(summary, step)
print('Epoch %d: Average loss is: %.5f; Average accuracy is: %.5f'%(epoch+1, loss_epoch / (len(train_video_indices) // BATCH_SIZE),
accuracy_epoch / (len(train_video_indices) // BATCH_SIZE)))
accuracy_epoch = 0
loss_epoch = 0
logits = C3D_model.C3D(batch_clips, NUM_CLASSES, 0.5)
accuracy = tf.reduce_mean(
tf.cast(tf.equal(tf.argmax(logits, 1), tf.argmax(batch_labels, 1)),
np.float32))
restorer = tf.train.Saver()
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
restorer.restore(sess, TRAIN_CHECK_POINT)
accuracy_epoch = 0
batch_index = 0
for i in range(test_num // BATCH_SIZE):
if i % 10 == 0:
print('Testing %d of %d' %
(i + 1, range(test_num // BATCH_SIZE)))
batch_data, batch_index = data_processing.get_batches(
TEST_LIST_PATH, NUM_CLASSES, batch_index, test_video_indices,
BATCH_SIZE)
accuracy_out = sess.run(accuracy,
feed_dict={
batch_clips: batch_data['clips'],
batch_labels: batch_data['labels']
})
accuracy_epoch += accuracy_out
print('Test accuracy is %.5f' % (accuracy_epoch /
(test_num // BATCH_SIZE)))
# ----------------------------------------
# Training
# ----------------------------------------
torch.backends.cudnn.benchmark=True
for epoch in range(opt.epochs):
# scheduler.step()
accuracy_epoch = 0
loss_epoch = 0
batch_index = 0
step = 0
for i in range(len(train_video_indices) // opt.batch_size):
step += 1
batch_data, batch_index = data_processing.get_batches(TRAIN_LIST_PATH, opt.num_classes, batch_index,
train_video_indices, opt.batch_size)
data = batch_data['clips']
data = data.transpose(0, 4, 1, 2, 3)
data = torch.from_numpy(data)
data = data.type(torch.cuda.FloatTensor)
target = batch_data['labels']
target = torch.from_numpy(target)
target = target.type(torch.cuda.LongTensor)
if cuda:
data = data.cuda()
target = target.cuda()
optimizer.zero_grad()
out = model(data)
loss = criterion(out, target)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.Session() as sess:
train_writer = tf.summary.FileWriter(CHECK_POINT_PATH, sess.graph)
# Restore variables from disk.
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
restorer.restore(sess, MODEL_PATH)
#Obtain style image batch
style_image_batch = data_processing.read_image(STYLE_IMAGE_PATH, True,
BATCH_SIZE)
step = 0
for epoch in range(EPOCH_NUM):
batch_index = 0
for i in range(DATA_SIZE // BATCH_SIZE):
image_batch, batch_index = data_processing.get_batches(
TRAIN_IMAGE_PATH, batch_index, BATCH_SIZE)
_, batch_ls, style_ls, content_ls, summary_str = sess.run(
[opt_op, loss, style_loss, content_loss, summary],
feed_dict={
image: image_batch,
style_image: style_image_batch
})
step += 1
if i % 10 == 0:
print(
'Epoch %d, Batch %d of %d, loss is %.3f, style loss is %.3f, content loss is %.3f'
% (epoch + 1, i, DATA_SIZE // BATCH_SIZE, batch_ls,
220 * style_ls, content_ls))
train_writer.add_summary(summary_str, step)
test_image = data_processing.read_image(TEST_IMAGE_PATH)
styled_image = sess.run(squeezed_generated_image,
def main(nets_archi, train_data, test_data, mode_, name="test"):
# Preprocessing data
data_size = train_data.shape[0]
# Create weights DST dir
DST = create_DST(name)
###### Reset tf graph ######
tf.reset_default_graph()
start_time = time.time()
print("\nPreparing variables and building model ...")
###### Create tf placeholder for obs variables ######
y = tf.placeholder(dtype=data_type(),
shape=(None, IMAGE_SIZE, IMAGE_SIZE, 1))
normal_mean = tf.placeholder(dtype=data_type(), shape=(K, N, N + 1))
###### Create varaible for batch ######
batch = tf.Variable(0, dtype=data_type())
###### CLearning rate decay ######
learning_rate = tf.train.exponential_decay(
learning_rate_init, # Base learning rate.
batch * BATCH_SIZE, # Current index into the dataset.
20 * data_size, # Decay step.
0.97, # Decay rate.
staircase=True)
###### Create instance SVAE ######
recognition_net = nets_archi["recog"]
generator_net = nets_archi["gener"]
svae_ = svae.SVAE(
recog_archi=recognition_net, # architecture of the recognition network
gener_archi=generator_net, # architecture of the generative network
K=K, # dim of the discrete latents z
N=N, # dim of the gaussian latents x
P=IMAGE_SIZE * IMAGE_SIZE, # dim of the obs variables y
max_iter=niter) # number of iterations in the coordinate block ascent
###### Initialize parameters ######
labels_stats_init_tiled, label_global_mean, gauss_global_mean = svae_._init_params(
)
# We need to tile the natural parameters for each inputs in batch (inputs are iid)
tile_shape = [BATCH_SIZE, 1, 1, 1]
gauss_global_mean_tiled = tf.tile(
tf.expand_dims(gauss_global_mean,
0), tile_shape) # shape: [batch,n_mixtures,dim,1+dim]
label_global_mean_tiled = tf.tile(tf.expand_dims(
label_global_mean, 0), tile_shape[:-1]) # shape: [batch,n_mixtures,1]
# We convert the global mean parameters to global natural parameters
gaussian_global = svae_.gaussian.standard_to_natural(
gauss_global_mean_tiled)
label_global = svae_.labels.standard_to_natural(label_global_mean_tiled)
###### Build loss and optimizer ######
svae_._create_loss_optimizer(gaussian_global, label_global,
labels_stats_init_tiled, y, learning_rate,
batch)
###### Build generator ######
svae_._generate(
tf.tile(tf.expand_dims(normal_mean, 0), [nexamples, 1, 1, 1]))
###### Initializer ######
init = tf.global_variables_initializer()
###### Saver ######
saver = tf.train.Saver()
###### Create a local session to run the training ######
with tf.Session() as sess:
# Training
if mode_ == "training":
# Opening csv file
csv_path = "./Perf"
if not tf.gfile.Exists(csv_path):
os.makedirs(csv_path)
csvfileTrain = open(os.path.join(csv_path, name) + ".csv", 'w')
Trainwriter = csv.writer(
csvfileTrain,
delimiter=';',
)
Trainwriter.writerow(['Num Epoch', 'train loss', 'test_loss'])
# Initialize variables
sess.run(tf.global_variables_initializer())
# initialize performance indicators
best_l = -10000000000.0
#training loop
print("\nStart training ...")
for epoch in range(num_epochs):
start_time = time.time()
print("")
print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))
# Training loop
train_l = 0.0
batches = data_processing.get_batches(train_data, BATCH_SIZE)
for i, batch in enumerate(batches):
_, l, lr = sess.run(
[svae_.optimizer, svae_.SVAE_obj, learning_rate],
feed_dict={y: batch})
# Update average loss
train_l += l / len(batches)
# Testing loop
test_l = 0.0
batches = data_processing.get_batches(test_data, BATCH_SIZE)
for i, batch in enumerate(batches):
l = sess.run(svae_.SVAE_obj, feed_dict={y: batch})
# Update average loss
test_l += l / len(batches)
# Update best perf and save model
if test_l > best_l:
best_l = test_l
if epoch > 20:
saver.save(sess, DST)
print("model saved.")
# Print info for previous epoch
print("Epoch {} done, took {:.2f}s, learning rate: {:10.2e}".
format(epoch,
time.time() - start_time, lr))
print(
"Train loss: {:.3f}, Test loss: {:.3f},Best test loss: {:.3f}"
.format(train_l, test_l, best_l))
# Writing csv file with results and saving models
Trainwriter.writerow([epoch + 1, train_l, test_l])
if mode_ == "reconstruct":
#Plot reconstruction mean
if not tf.gfile.Exists(DST + ".meta"):
raise Exception("no weights given")
saver.restore(sess, DST)
img = test_data[np.random.randint(0,
high=test_data.shape[0],
size=BATCH_SIZE)]
bernouilli_mean = sess.run(svae_.y_reconstr_mean,
feed_dict={y: img})
save_reconstruct(img[:nexamples], bernouilli_mean[:nexamples],
"./reconstruct")
if mode_ == "generate":
#Test for ploting images
if not tf.gfile.Exists(DST + ".meta"):
raise Exception("no weights given")
saver.restore(sess, DST)
gaussian_mean = sess.run(gauss_global_mean, feed_dict={})
#pdb.set_trace()
bernouilli_mean = sess.run(svae_.y_generate_mean,
feed_dict={normal_mean: gaussian_mean})
bernouilli_mean = np.transpose(
np.reshape(bernouilli_mean,
(nexamples, K, IMAGE_SIZE, IMAGE_SIZE)),
(1, 0, 2, 3))
save_gene(bernouilli_mean, "./generate")
def main(_):
with tf.device('/gpu:0'):
for regularization_type in ['Blackout', 'None', 'L1', 'L2']:
dataset_sizes = np.linspace(2500, 55000, num=22)
for size in dataset_sizes:
# Getting the appropriate dataset
print(int(size))
train_x, train_y, valid_x, valid_y, test_x, test_y = split_data(
dataset, int(size))
# Resetting the graph incase of multiple runs on the same console
tf.reset_default_graph()
for i in range(numOfTests):
num_layers = random.choice([5, 6, 7, 8, 9, 10])
num_nodes = random.choice([200, 400, 600])
num_inputs = int(train_x.shape[1])
num_steps = random.choice([50, 100, 150, 200])
regularization_scale = random.choice(
[0.01, 0.005, 0.001, 0.0005])
percent_connections_kept = random.choice([0.9, 0.95, 0.85])
num_classes = len(np.unique(train_y))
print('Test No. ' + str(i) + '/' + str(numOfTests))
print('Parameters: ' + str(size) + ',' +
regularization_type + ',' + str(num_layers) + ',' +
str(num_nodes) + ',' + str(num_steps) + ',' +
str(regularization_scale) + ',' +
str(percent_connections_kept))
# Create the model
x = tf.placeholder(tf.float32, [None, num_inputs])
y = create_model(x, num_layers, num_nodes, num_classes)
# Define loss and optimizer
y_ = tf.placeholder(tf.int64, [None])
# Retrieving weights and defining regularization penalty
weights = tf.trainable_variables()
regularization_penalty, blackout_weights = get_regularization_penalty(
weights, regularization_scale,
percent_connections_kept, regularization_type)
# Defining loss and optimizer
cross = tf.losses.sparse_softmax_cross_entropy(labels=y_,
logits=y)
loss = cross + regularization_penalty
train_step = tf.train.RMSPropOptimizer(0.001).minimize(
loss)
# Evaluate Model
correct_prediction = tf.equal(tf.argmax(y, 1), y_)
accuracy = tf.reduce_mean(
tf.cast(correct_prediction, tf.float32))
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
# Initializing session
sess = tf.InteractiveSession(config=config)
tf.global_variables_initializer().run()
# Train
# PercentageOfConnOff=[]
# LossFunctionRegu=[]
# LossFunctionCrossTrain=[]
# LossFunctionCrossValid=[]
#
numOfBatches = 50
all_batches_x, all_batches_y = get_batches(
train_x, train_y, numOfBatches)
# Train
for i in range(num_steps):
randomPick = random.randint(0, numOfBatches)
#print(str(len(all_batches_x)) + " getting " + str(randomPick))
if randomPick == 50:
randomPick = 49
currentBatchX = all_batches_x[randomPick]
currentBatchY = all_batches_y[randomPick]
sess.run(train_step,
feed_dict={
x: currentBatchX,
y_: currentBatchY
})
# Test trained model
if i % 20 == 1:
print('Accuracy: ' + str(
sess.run(accuracy,
feed_dict={
x: valid_x,
y_: valid_y
})))
# if regularization_type=='Blackout':
# currentWeights=sess.run(blackout_weights)
# part1=currentWeights>-0.01
# part2=currentWeights<0.01
# turnedOff=np.sum(np.logical_and(part1,part2))
# TotalNumOfWeights=float(currentWeights.shape[0])
# LossFunctionCrossTrain.append(sess.run(cross, feed_dict={x: train_x, y_: train_y}))
# LossFunctionCrossValid.append(sess.run(cross, feed_dict={x: valid_x, y_: valid_y}))
# LossFunctionRegu.append(sess.run(regularization_penalty))
# PercentageOfConnOff.append((TotalNumOfWeights-turnedOff)/TotalNumOfWeights)
#if regularization_type=='Blackout':
# fig = plt.figure()
# ax1 = fig.add_subplot(1, 2, 1)
# ax2 = fig.add_subplot(1, 2, 2)
# ax1.plot(PercentageOfConnOff)
# ax2.plot(LossFunctionCrossTrain,label='Cross-Entropy Train')
# ax2.plot(LossFunctionCrossValid,label='Cross-Entropy Validation')
# ax2.plot(LossFunctionRegu,label='Regularization')
# ax2.legend()
# fig.show()
accuracyVal = sess.run(accuracy,
feed_dict={
x: valid_x,
y_: valid_y
})
accuracyTest = sess.run(accuracy,
feed_dict={
x: test_x,
y_: test_y
})
tf.reset_default_graph()
store_results(dataset, regularization_type, num_layers,
num_nodes, num_steps, regularization_scale,
percent_connections_kept, accuracyVal,
accuracyTest, size)
print('Accuracy Val: ' + str(accuracyVal) +
' , Accuracy Test: ' + str(accuracyTest))
def read_labels_from_file(filepath):
with open(filepath, 'r') as f:
labels = [line.strip() for line in f.readlines()]
return labels
test_path = 'test_cross.list'
torch.backends.cudnn.benchmark = True
if __name__ == "__main__":
with open(test_path, 'r') as t:
test_num = len(list(t))
test_video_indices = range(test_num)
batch_index = 0
for i in range(test_num // opt.batch_size):
batch_data, batch_index = data_processing.get_batches(
test_path, opt.num_classes, batch_index, test_video_indices,
opt.batch_size)
clip = torch.from_numpy(batch_data['clips'].transpose(0, 4, 1, 2, 3))
clip = clip.cuda()
net = C3D_model_pytorch.C3D(dropout_rate=1)
net.load_state_dict(torch.load('C3D_model_pytorch.pkl'))
net.cuda()
net.eval()
output = net(clip)
labels = torch.from_numpy(np.array(batch_data['labels']))
output_index = torch.max(output, 1)[1].cpu().numpy()
equ_num = 0
for i in range(opt.batch_size):