def main(gpu_id=None):
if gpu_id is not None:
os.environ["CUDA_VISIBLE_DEVICES"] = gpu_id
# Reset Tensorflow graph
tf.reset_default_graph()
# Generate stimulus
stim = Stimulus()
# Load saved genetically trained model
evo_model = EvoModel()
saved_evo_model = pickle.load(
open('./savedir/conv_task/run_21_model_stats.pkl', 'rb'))
evo_model.update_variables(saved_evo_model['var_dict'])
print('Loaded evo model')
config = tf.ConfigProto(allow_soft_placement=True)
with tf.Session(config=config) as sess:
init = tf.global_variables_initializer()
sess.run(init)
device = '/cpu:0' if gpu_id is None else '/gpu:0'
with tf.device(device):
# Load trained convolutional autoencoder
if par['task'] == 'conv_task':
folder = './latent_all_img_batch16_filt16_loss80/'
conv_model = tf.train.import_meta_graph(
folder + 'conv_model_with_latent.meta', clear_devices=True)
conv_model.restore(sess, tf.train.latest_checkpoint(folder))
print('Loaded conv model from', folder)
# Generate batch and save output images
for i in range(4):
input_data, conv_target, evo_target = stim.generate_train_batch(
)
# Run input through convolutional model
feed_dict = {'x:0': input_data, 'y:0': conv_target}
test_loss, conv_output, encoded = sess.run(
['l:0', 'o:0', 'encoded:0'], feed_dict=feed_dict)
# Run latent output through evolutionary model
evo_model.load_batch(encoded, evo_target)
evo_model.run_models()
evo_model.judge_models()
# Save output from both models
plot_conv_evo_outputs(conv_target,
conv_output,
evo_target,
evo_model.output,
i,
test=True)
else:
folder = './'
conv_top = tf.train.import_meta_graph(folder +
'conv_model_top.meta',
clear_devices=True)
conv_top.restore(sess, tf.train.latest_checkpoint(folder))
for i in range(4):
test_input, test_target, _ = stim.generate_test_batch()
feed_dict = {
'input:0': test_input,
'target:0': test_target
}
test_loss, test_output = sess.run(['l:0', 'output:0'],
feed_dict=feed_dict)
plot_conv_all(test_target, test_output, i)
def main(gpu_id = None):
if gpu_id is not None:
os.environ["CUDA_VISIBLE_DEVICES"] = gpu_id
# Generate stimulus
stim = Stimulus()
# Initialize evolutionary model
evo_model = EvoModel()
# Model stats
losses = []
testing_losses = []
save_iter = []
# Reset Tensorflow graph
tf.reset_default_graph()
config = tf.ConfigProto(allow_soft_placement=True)
with tf.Session(config=config) as sess:
init = tf.global_variables_initializer()
sess.run(init)
device = '/cpu:0' if gpu_id is None else '/gpu:0'
with tf.device(device):
# Load saved convolutional autoencoder
folder = './latent_all_img_batch16_filt16_loss80/'
conv_model = tf.train.import_meta_graph(folder + 'conv_model_with_latent.meta', clear_devices=True)
conv_model.restore(sess, tf.train.latest_checkpoint(folder))
print('Loaded model from',folder)
stuck = 0
test_loss = [1000000]
threshold = [10000, 5000, 1000, 750, 500, 300, 150, -1]
# Train the model
start = time.time()
for i in range(par['num_iterations']):
# Generate train batch and run through the convolutional autoencoder
input_data, conv_target, evo_target = stim.generate_train_batch()
feed_dict = {'x:0': input_data, 'y:0': conv_target}
conv_loss, conv_output, encoded = sess.run(['l:0', 'o:0','encoded:0'], feed_dict=feed_dict)
# One cycle of evolutionary model
evo_model.load_batch(encoded, evo_target)
evo_model.run_models()
evo_model.judge_models()
evo_model.breed_models_genetic()
# If all models are performing poorly, introduce new randomly generated models
evo_loss = evo_model.get_losses(True)
if par['num_migrators']:
evo_model.migration()
if evo_loss[0] < 10000:
par['num_migrators'] = 0
# Decrease mutation rate when loss value is below certain threshold
if len(threshold) > 0 and evo_loss[0] < threshold[0]:
evo_model.slowdown_mutation()
threshold.pop(0)
# If there is no improvement in performance for many iterations, change mutation rate
if evo_loss[0] < test_loss[0]:
stuck = 0
else:
stuck += 1
if stuck > 20:
# evo_model.speed_up_mutation()
evo_model.slowdown_mutation()
stuck = 0
# Check current status
if i % par['print_iter'] == 0:
# Print current status
print_evo_stats(i, evo_model.con_dict['mutation_rate'], evo_model.con_dict['mutation_strength'], stuck, conv_loss, np.array([*evo_loss[0:3],evo_loss[par['num_survivors']-1]]), time.time()-start)
losses.append(evo_loss[0])
save_iter.append(i)
# Save model and output
if i % par['save_iter'] == 0 and (evo_loss[0] < test_loss[0] or i%100 == 0):
# Generate batch from testing set and run through both models
input_data, conv_target, evo_target = stim.generate_test_batch()
feed_dict = {'x:0': input_data, 'y:0': conv_target}
test_loss, conv_output, encoded = sess.run(['l:0', 'o:0','encoded:0'], feed_dict=feed_dict)
evo_model.load_batch(encoded, evo_target)
evo_model.run_models()
evo_model.judge_models()
test_loss = evo_model.get_losses(True)
testing_losses.append(test_loss[0])
# Save output images
plot_conv_evo_outputs(conv_target, conv_output, evo_target, evo_model.output, i)
# Save model
pickle.dump({'iter':save_iter,'var_dict':evo_model.var_dict, 'losses': losses, 'test_loss': testing_losses, 'last_iter': i}, \
open(par['save_dir']+'run_'+str(par['run_number'])+'_model_stats.pkl', 'wb'))
# Plot loss curve
if i > 0:
plt.plot(losses[1:])
plt.savefig(par['save_dir']+'run_'+str(par['run_number'])+'_training_curve.png')
plt.close()
def main(gpu_id=None):
if gpu_id is not None:
os.environ["CUDA_VISIBLE_DEVICES"] = gpu_id
# Reset Tensorflow graph
tf.reset_default_graph()
# Placeholders for the tensorflow model
x = tf.placeholder(tf.float32,
shape=[par['batch_train_size'], par['n_input']],
name='x')
y = tf.placeholder(tf.float32,
shape=[par['batch_train_size'], par['n_output']],
name='y')
# Generate stimulus
stim = Stimulus()
# Model stats
losses = []
testing_losses = []
save_iter = []
config = tf.ConfigProto()
with tf.Session(config=config) as sess:
device = '/cpu:0' if gpu_id is None else '/gpu:0'
with tf.device(device):
model = Model(x, y)
init = tf.global_variables_initializer()
sess.run(init)
saver = tf.train.Saver()
# Train the model
prev_loss = 10000000
start = time.time()
for i in range(par['num_iterations']):
# Generate training batch and train model
input_data, target_data, _ = stim.generate_train_batch()
feed_dict = {x: input_data, y: target_data}
_, train_loss, model_output = sess.run(
[model.train_op, model.loss, model.output],
feed_dict=feed_dict)
# Check current status
if i % par['print_iter'] == 0:
# Print current status
print_conv_stats(i, train_loss, time.time() - start)
losses.append(train_loss)
save_iter.append(i)
# Test and save model
if i % par['save_iter'] == 0:
# Generate test bach and get model performance
test_input, test_target, _ = stim.generate_test_batch()
feed_dict = {x: test_input, y: test_target}
test_loss, test_output = sess.run([
model.loss,
model.output,
],
feed_dict=feed_dict)
testing_losses.append(test_loss)
# Plot model outputs
if test_loss < prev_loss:
prev_loss = test_loss
plot_conv_outputs(target_data, model_output,
test_target, test_output, i)
# Save training stats and model
weight = sess.run(
tf.get_collection(tf.GraphKeys.VARIABLES,
'filters/kernel')[0])
pickle.dump({'weight':weight,'iter':save_iter,'losses': losses, 'test_loss': testing_losses, 'last_iter': i}, \
open(par['save_dir']+'run_'+str(par['run_number'])+'_model_stats.pkl', 'wb'))
saved_path = saver.save(
sess, './upsample2/conv_model_with_latent')
print('model saved in {}'.format(saved_path))
# Plot loss curve
if i > 0:
plt.plot(losses[1:])
plt.savefig(par['save_dir'] + 'run_' +
str(par['run_number']) + '_training_curve.png')
plt.close()
def main(gpu_id = None):
if gpu_id is not None:
os.environ["CUDA_VISIBLE_DEVICES"] = gpu_id
# Reset Tensorflow graph
tf.reset_default_graph()
# Generate stimulus
stim = Stimulus()
# Placeholders for the tensorflow model
x = tf.placeholder(tf.float32, shape=[par['batch_train_size'],par['n_input']])
y = tf.placeholder(tf.float32, shape=[par['batch_train_size'],par['n_output']])
# Model stats
losses = []
testing_losses = []
save_iter = []
prev_loss = 10000
config = tf.ConfigProto()
with tf.Session(config=config) as sess:
device = '/cpu:0' if gpu_id is None else '/gpu:0'
with tf.device(device):
model = Model(x,y)
init = tf.global_variables_initializer()
sess.run(init)
# Train the model
start = time.time()
for i in range(par['num_iterations']):
# Generate training set
input_data, target_data, _ = stim.generate_train_batch()
feed_dict = {x: input_data, y: target_data}
_, train_loss, model_output = sess.run([model.train_op, model.loss, model.output], feed_dict=feed_dict)
# Check current status
if i % par['print_iter'] == 0:
# Print current status
print('Model {:2} | Task: {:s} | Iter: {:6} | Loss: {:8.3f} | Run Time: {:5.3f}s'.format( \
par['run_number'], par['task'], i, train_loss, time.time()-start))
losses.append(train_loss)
save_iter.append(i)
# Save one training and output img from this iteration
if i % par['save_iter'] == 0:
# Generate batch from testing set and check the output
test_input, test_target, _ = stim.generate_test_batch()
feed_dict = {x: test_input, y: test_target}
test_loss, test_output = sess.run([model.loss, model.output], feed_dict=feed_dict)
testing_losses.append(test_loss)
if test_loss < prev_loss:
prev_loss = test_loss
# Results from a training sample
original1 = target_data[0].reshape(par['out_img_shape'])
output1 = model_output[0].reshape(par['out_img_shape'])
font = cv2.FONT_HERSHEY_SIMPLEX
cv2.putText(original1,'Training',(5,20), font, 0.5,(255,255,255), 2, cv2.LINE_AA)
cv2.putText(output1,'Output',(5,20), font, 0.5,(255,255,255), 2, cv2.LINE_AA)
# Results from a testing sample
original2 = test_target[1].reshape(par['out_img_shape'])
output2 = test_output[1].reshape(par['out_img_shape'])
original3 = test_target[2].reshape(par['out_img_shape'])
output3 = test_output[2].reshape(par['out_img_shape'])
cv2.putText(original2,'Testing',(5,20), font, 0.5,(255,255,255), 2, cv2.LINE_AA)
cv2.putText(output2,'Output',(5,20), font, 0.5,(255,255,255), 2, cv2.LINE_AA)
vis1 = np.concatenate((original1, output1), axis=1)
vis2 = np.concatenate((original2, output2), axis=1)
vis3 = np.concatenate((original3, output3), axis=1)
vis = np.concatenate((vis1, vis2), axis=0)
vis = np.concatenate((vis, vis3), axis=0)
cv2.imwrite(par['save_dir']+'run_'+str(par['run_number'])+'_test_'+str(i)+'.png', vis)
pickle.dump({'iter':save_iter,'losses': losses, 'test_loss': testing_losses, 'last_iter': i}, \
open(par['save_dir']+'run_'+str(par['run_number'])+'_model_stats.pkl', 'wb'))
# Plot loss curve
if i > 0:
plt.plot(losses[1:])
plt.savefig(par['save_dir']+'run_'+str(par['run_number'])+'_training_curve.png')
plt.close()
# Stop training if loss is small enough (just for sweep purposes)
if train_loss < 30:
break