def valid(self,):
print 'Performing validation.'
model = RNN_RNADE(self.state['n_visible'],self.state['n_hidden'],self.state['n_recurrent'],self.state['n_components'],hidden_act=self.state['hidden_act'],
l2=self.state['l2'],rec_mu=self.state['rec_mu'],rec_mix=self.state['rec_mix'],rec_sigma=self.state['rec_sigma'],load=False,load_dir=self.output_folder)
#model.params = self.params
model.load_model(self.output_folder,'best_params_train.pickle')
num_test_sequences = 1
batch_size = 100
num_samples = 1
error = []
for i in xrange(num_test_sequences):
seq = b.bounce_vec(15,n=3,T=128)
samples = model.sample_given_sequence(seq,num_samples)
#make sure samples are between 0 and 1
samples = numpy.minimum(samples,1.)
samples = numpy.maximum(samples,0.)
sq_diff = (samples - seq)**2
sq_diff = sq_diff.mean(axis=0)
sq_diff = sq_diff.sum(axis=1)
seq_error = sq_diff.mean(axis=0)
error.append(seq_error)
total_error = numpy.mean(error)
print 'Validation error: ',total_error
self.valid_costs.append(total_error)
if total_error < self.best_valid_cost:
print 'Best validation params!'
self.best_valid_cost = total_error
self.save_model('best_params_valid.pickle')
def generate_bouncing_ball_sample(batch_size, seq_length, shape, num_balls):
"""
Creates sequences of bouncing balls.
Args:
batch_size: integer, number of sequences to generate
seq_length: number of frames to generate for each sequence
shape: [m, n, k] list where m and n are the frame height and width,
and k is the number of channels ... *** note: m must = n!
num_balls: number of balls to generate
"""
dat = np.zeros((batch_size, seq_length, shape[0], shape[1], 3))
for i in xrange(batch_size):
dat[i, :, :, :, :] = b.bounce_vec(shape[0], num_balls, seq_length)
if shape[2] == 1:
rgb_dat = np.zeros((batch_size, seq_length, shape[0], shape[1], 1))
rgb_dat[:, :, :, :, 0] = np.dot(dat[..., :3],
[0.299, 0.587, 0.114]) #make grayscale
return rgb_dat
return dat
def valid(self, ):
print 'Performing validation.'
model = RNN_RNADE(self.state['n_visible'],
self.state['n_hidden'],
self.state['n_recurrent'],
self.state['n_components'],
hidden_act=self.state['hidden_act'],
l2=self.state['l2'],
rec_mu=self.state['rec_mu'],
rec_mix=self.state['rec_mix'],
rec_sigma=self.state['rec_sigma'],
load=False,
load_dir=self.output_folder)
#model.params = self.params
model.load_model(self.output_folder, 'best_params_train.pickle')
num_test_sequences = 1
batch_size = 100
num_samples = 1
error = []
for i in xrange(num_test_sequences):
seq = b.bounce_vec(15, n=3, T=128)
samples = model.sample_given_sequence(seq, num_samples)
#make sure samples are between 0 and 1
samples = numpy.minimum(samples, 1.)
samples = numpy.maximum(samples, 0.)
sq_diff = (samples - seq)**2
sq_diff = sq_diff.mean(axis=0)
sq_diff = sq_diff.sum(axis=1)
seq_error = sq_diff.mean(axis=0)
error.append(seq_error)
total_error = numpy.mean(error)
print 'Validation error: ', total_error
self.valid_costs.append(total_error)
if total_error < self.best_valid_cost:
print 'Best validation params!'
self.best_valid_cost = total_error
self.save_model('best_params_valid.pickle')
def generate_bouncing_ball_sample(batch_size, seq_length, shape, num_balls):
dat = np.zeros((batch_size, seq_length, shape, shape, 3))
for i in range(batch_size):
dat[i, :, :, :, :] = b.bounce_vec(32, num_balls, seq_length)
return dat
def generate_bouncing_ball_sample(batch_size, seq_length, shape, num_balls):
dat = np.zeros((batch_size, seq_length, shape, shape, 3))
for i in range(batch_size):
dat[i, :, :, :, :] = b.bounce_vec(64, num_balls, seq_length)
return dat
def train():
"""Train ring_net for a number of steps."""
with tf.Graph().as_default():
# make inputs
x = tf.placeholder(tf.float32, [FLAGS.batch_size, 32, 32, 1])
# possible dropout inside (default is 1.0)
keep_prob = tf.placeholder("float")
# make model
if FLAGS.model=="fully_connected":
mean, stddev, y_sampled, x_prime = arc.fully_connected_model(x, keep_prob)
elif FLAGS.model=="conv":
mean, stddev, y_sampled, x_prime = arc.conv_model(x, keep_prob)
elif FLAGS.model=="all_conv":
mean, stddev, y_sampled, x_prime = arc.all_conv_model(x, keep_prob)
else:
print("model requested not found, now some error!")
# calc loss stuff
loss_vae, loss_reconstruction, loss, train_op = ls.loss(mean, stddev, x, x_prime)
# List of all Variables
variables = tf.all_variables()
# Build a saver
saver = tf.train.Saver(tf.all_variables())
# Summary op
summary_op = tf.merge_all_summaries()
# Build an initialization operation to run below.
init = tf.initialize_all_variables()
# Start running operations on the Graph.
sess = tf.Session()
# init if this is the very time training
print("init network from scratch")
sess.run(init)
# Summary op
graph_def = sess.graph.as_graph_def(add_shapes=True)
summary_writer = tf.train.SummaryWriter(train_dir_save, graph_def=graph_def)
for step in xrange(FLAGS.max_step):
dat = b.bounce_vec(32, FLAGS.num_balls, FLAGS.batch_size)
t = time.time()
_, loss_r = sess.run([train_op, loss],feed_dict={x:dat, keep_prob:FLAGS.keep_prob})
elapsed = time.time() - t
if step%2000 == 0:
_ , loss_vae_r, loss_reconstruction_r, y_sampled_r, x_prime_r, stddev_r = sess.run([train_op, loss_vae, loss_reconstruction, y_sampled, x_prime, stddev],feed_dict={x:dat, keep_prob:FLAGS.keep_prob})
summary_str = sess.run(summary_op, feed_dict={x:dat, keep_prob:FLAGS.keep_prob})
summary_writer.add_summary(summary_str, step)
print("loss vae value at " + str(loss_vae_r))
print("loss reconstruction value at " + str(loss_reconstruction_r))
print("time per batch is " + str(elapsed))
cv2.imwrite("real_balls.jpg", np.uint8(dat[0, :, :, :]*255))
cv2.imwrite("generated_balls.jpg", np.uint8(x_prime_r[0, :, :, :]*255))
stddev_r = np.sort(np.sum(stddev_r, axis=0))
plt.plot(stddev_r/FLAGS.batch_size, label="step " + str(step))
plt.legend()
plt.savefig('stddev_num_balls_' + str(FLAGS.num_balls) + '_beta_' + str(FLAGS.beta) + '.png')
assert not np.isnan(loss_r), 'Model diverged with loss = NaN'
if step%1000 == 0:
checkpoint_path = os.path.join(train_dir_save, 'model.ckpt')
saver.save(sess, checkpoint_path, global_step=step)
print("saved to " + train_dir_save)
print("step " + str(step))
def train():
"""Train ring_net for a number of steps."""
with tf.Graph().as_default():
# make inputs
x = tf.placeholder(tf.float32, [FLAGS.batch_size, 32, 32, 3])
# possible dropout inside
keep_prob = tf.placeholder("float")
# create network
# encodeing part first
# conv1
conv1 = ld.conv_layer(x, 3, 2, 8, "encode_1")
# conv2
conv2 = ld.conv_layer(conv1, 3, 1, 8, "encode_2")
# conv3
conv3 = ld.conv_layer(conv2, 3, 2, 8, "encode_3")
# conv4
conv4 = ld.conv_layer(conv3, 1, 1, 4, "encode_4")
# fc5
fc5 = ld.fc_layer(conv4, 128, "encode_5", True, False)
# dropout maybe
fc5_dropout = tf.nn.dropout(fc5, keep_prob)
# y
y = ld.fc_layer(fc5_dropout, (FLAGS.hidden_size) * 2, "encode_6",
False, True)
mean, stddev = tf.split(1, 2, y)
stddev = tf.sqrt(tf.exp(stddev))
# now decoding part
# sample distrobution
epsilon = tf.random_normal(mean.get_shape())
y_sampled = mean + epsilon * stddev
# fc7
fc7 = ld.fc_layer(y_sampled, 128, "decode_7", False, False)
# fc8
fc8 = ld.fc_layer(fc7, 4 * 8 * 8, "decode_8", False, False)
conv9 = tf.reshape(fc8, [-1, 8, 8, 4])
# conv10
conv10 = ld.transpose_conv_layer(conv9, 1, 1, 8, "decode_9")
# conv11
conv11 = ld.transpose_conv_layer(conv10, 3, 2, 8, "decode_10")
# conv12
conv12 = ld.transpose_conv_layer(conv11, 3, 1, 8, "decode_11")
# conv13
conv13 = ld.transpose_conv_layer(conv12, 3, 2, 3, "decode_12", True)
# x_prime
x_prime = conv13
x_prime = tf.nn.sigmoid(x_prime)
# now calc loss
epsilon = 1e-8
# calc loss from vae
kl_loss = 0.5 * (tf.square(mean) + tf.square(stddev) -
2.0 * tf.log(stddev + epsilon) - 1.0)
loss_vae = FLAGS.beta * tf.reduce_sum(kl_loss)
# log loss for reconstruction
loss_reconstruction = tf.reduce_sum(-x * tf.log(x_prime + epsilon) -
(1.0 - x) *
tf.log(1.0 - x_prime + epsilon))
# save for tensorboard
tf.scalar_summary('loss_vae', loss_vae)
tf.scalar_summary('loss_reconstruction', loss_reconstruction)
# calc total loss
loss = tf.reduce_sum(loss_vae + loss_reconstruction)
# training
train_op = tf.train.AdamOptimizer(FLAGS.lr).minimize(loss)
# List of all Variables
variables = tf.all_variables()
# Build a saver
saver = tf.train.Saver(tf.all_variables())
# Summary op
summary_op = tf.merge_all_summaries()
# Build an initialization operation to run below.
init = tf.initialize_all_variables()
# Start running operations on the Graph.
sess = tf.Session()
# init if this is the very time training
print("init network from scratch")
sess.run(init)
# Summary op
graph_def = sess.graph.as_graph_def(add_shapes=True)
summary_writer = tf.train.SummaryWriter(FLAGS.train_dir,
graph_def=graph_def)
for step in xrange(FLAGS.max_step):
dat = b.bounce_vec(32, FLAGS.num_balls, FLAGS.batch_size)
t = time.time()
_, loss_r = sess.run([train_op, loss],
feed_dict={
x: dat,
keep_prob: FLAGS.keep_prob
})
elapsed = time.time() - t
#print(elapsed)
if step % 500 == 0:
_, loss_vae_r, loss_reconstruction_r, y_sampled_r, x_prime_r, kl_loss_dis, stddev_r = sess.run(
[
train_op, loss_vae, loss_reconstruction, y_sampled,
x_prime, kl_loss, stddev
],
feed_dict={
x: dat,
keep_prob: FLAGS.keep_prob
})
summary_str = sess.run(summary_op,
feed_dict={
x: dat,
keep_prob: FLAGS.keep_prob
})
summary_writer.add_summary(summary_str, step)
print("loss vae value at " + str(loss_vae_r))
print("loss reconstruction value at " +
str(loss_reconstruction_r))
print("min sampled vector " + str(np.min(y_sampled_r)))
print("max sampled vector " + str(np.max(y_sampled_r)))
print("time per batch is " + str(elapsed))
cv2.imwrite("real_balls.jpg", np.uint8(dat[0, :, :, :] * 255))
cv2.imwrite("generated_balls.jpg",
np.uint8(x_prime_r[0, :, :, :] * 255))
kl_loss_dis = np.sort(np.sum(kl_loss_dis, axis=0))
stddev_r = np.sort(np.sum(stddev_r, axis=0))
#plt.plot(kl_loss_dis, label="step " + str(step))
#plt.legend(loc = 'center left')
#plt.savefig('kl_error_dis.png')
plt.plot(stddev_r, label="step " + str(step))
plt.legend(loc='center left')
plt.savefig('stddev_r.png')
assert not np.isnan(loss_r), 'Model diverged with loss = NaN'
if step % 1000 == 0:
checkpoint_path = os.path.join(FLAGS.train_dir, 'model.ckpt')
saver.save(sess, checkpoint_path, global_step=step)
print("saved to " + FLAGS.train_dir)
print("step " + str(step))
def test_stddev(network_type):
# set parameters (quick and dirty code just to make graphs fast)
if network_type in ("model_conv_num_balls_1_beta_0.1",
"model_conv_num_balls_1_beta_0.5",
"model_conv_num_balls_1_beta_1.0"):
FLAGS.model = "conv"
FLAGS.num_balls = 1
if network_type in ("model_conv_num_balls_2_beta_0.1",
"model_conv_num_balls_2_beta_0.5",
"model_conv_num_balls_2_beta_1.0"):
FLAGS.model = "conv"
FLAGS.num_balls = 2
elif network_type in ("model_fully_connected_num_balls_1_beta_0.1",
"model_fully_connected_num_balls_1_beta_0.5",
"model_fully_connected_num_balls_1_beta_1.0"):
FLAGS.model = "fully_connected"
FLAGS.num_balls = 1
FLAGS.hidden_size = 10
elif network_type in ("model_fully_connected_num_balls_2_beta_0.1",
"model_fully_connected_num_balls_2_beta_0.5",
"model_fully_connected_num_balls_2_beta_1.0"):
FLAGS.model = "fully_connected"
FLAGS.num_balls = 2
FLAGS.hidden_size = 10
elif network_type in ("model_all_conv_num_balls_1_beta_0.1",
"model_all_conv_num_balls_1_beta_0.5",
"model_all_conv_num_balls_1_beta_1.0"):
FLAGS.model = "all_conv"
FLAGS.num_balls = 1
FLAGS.hidden_size = 1
elif network_type in ("model_all_conv_num_balls_2_beta_0.1",
"model_all_conv_num_balls_2_beta_0.5",
"model_all_conv_num_balls_2_beta_1.0"):
FLAGS.model = "all_conv"
FLAGS.num_balls = 2
FLAGS.hidden_size = 1
"""Eval net to get stddev."""
with tf.Graph().as_default():
# make inputs
x = tf.placeholder(tf.float32, [FLAGS.batch_size, 32, 32, 1])
# no dropout on testing
keep_prob = 1.0
# make model
if FLAGS.model == "fully_connected":
mean, stddev, y_sampled, x_prime = arc.fully_connected_model(
x, keep_prob)
elif FLAGS.model == "conv":
mean, stddev, y_sampled, x_prime = arc.conv_model(x, keep_prob)
elif FLAGS.model == "all_conv":
mean, stddev, y_sampled, x_prime = arc.all_conv_model(x, keep_prob)
else:
print("model requested not found, now some error!")
# List of all Variables
variables = tf.all_variables()
# Load weights operator
print('save file is ./checkpoints/train_store_' + network_type)
ckpt = tf.train.get_checkpoint_state('./checkpoints/train_store_' +
network_type)
weight_saver = tf.train.Saver(variables)
# Summary op
summary_op = tf.merge_all_summaries()
# Start running operations on the Graph.
sess = tf.Session()
# init if this is the very time training
weight_saver.restore(sess, ckpt.model_checkpoint_path)
print("restored from" + ckpt.model_checkpoint_path)
# Summary op
graph_def = sess.graph.as_graph_def(add_shapes=True)
dat = b.bounce_vec(32, FLAGS.num_balls, FLAGS.batch_size)
stddev_r = np.sort(
np.sum(sess.run([stddev], feed_dict={x: dat})[0], axis=0))
return stddev_r / FLAGS.batch_size
import bouncing_balls as b
import cv2
import numpy as np
res = 84
n_balls = 4
T = 200
dat = b.bounce_vec(res,n_balls,T)
b.show_V(dat)
print(dat.shape)
dat = dat.reshape(T, res, res)
dat = np.uint8(np.abs(dat * 255))
fourcc = cv2.cv.CV_FOURCC('m', 'p', '4', 'v')
video = cv2.VideoWriter()
success = video.open("test.mov", fourcc, 4, (84, 84), True)
for i in xrange(T-3):
frame = dat[i:i+3, :, :]
frame = np.transpose(frame, (1,2,0))
print(frame.shape)
video.write(frame)
def generate_bouncing_ball_sample(batch_size, seq_length, shape, num_balls):
dat = np.zeros((batch_size, seq_length, shape, shape, 3), dtype=np.float32)
for i in xrange(batch_size):
dat[i, :, :, :, :] = b.bounce_vec(32, num_balls, seq_length)
return torch.from_numpy(dat).permute(0, 1, 4, 2, 3)
def create_image(network_type):
# set parameters
if network_type in ("model_conv_num_balls_1_beta_0.1", "model_conv_num_balls_1_beta_0.5", "model_conv_num_balls_1_beta_1.0"):
FLAGS.model="conv"
FLAGS.num_balls=1
if network_type in ("model_conv_num_balls_2_beta_0.1", "model_conv_num_balls_2_beta_0.5", "model_conv_num_balls_2_beta_1.0"):
FLAGS.model="conv"
FLAGS.num_balls=2
elif network_type in ("model_fully_connected_num_balls_1_beta_0.1", "model_fully_connected_num_balls_1_beta_0.5", "model_fully_connected_num_balls_1_beta_1.0"):
FLAGS.model="fully_connected"
FLAGS.num_balls=1
FLAGS.hidden_size=10
elif network_type in ("model_fully_connected_num_balls_2_beta_0.1", "model_fully_connected_num_balls_2_beta_0.5", "model_fully_connected_num_balls_2_beta_1.0"):
FLAGS.model="fully_connected"
FLAGS.num_balls=2
FLAGS.hidden_size=10
elif network_type in ("model_all_conv_num_balls_1_beta_0.1", "model_all_conv_num_balls_1_beta_0.5", "model_all_conv_num_balls_1_beta_1.0"):
FLAGS.model="all_conv"
FLAGS.num_balls=1
FLAGS.hidden_size=1
elif network_type in ("model_all_conv_num_balls_2_beta_0.1", "model_all_conv_num_balls_2_beta_0.5", "model_all_conv_num_balls_2_beta_1.0"):
FLAGS.model="all_conv"
FLAGS.num_balls=2
FLAGS.hidden_size=1
"""Eval net to get stddev."""
with tf.Graph().as_default():
# make inputs
x = tf.placeholder(tf.float32, [FLAGS.batch_size, 32, 32, 1])
# no dropout on testing
keep_prob = 1.0
# make model
if FLAGS.model=="fully_connected":
mean, stddev, y_sampled, x_prime = arc.fully_connected_model(x, keep_prob)
elif FLAGS.model=="conv":
mean, stddev, y_sampled, x_prime = arc.conv_model(x, keep_prob)
elif FLAGS.model=="all_conv":
mean, stddev, y_sampled, x_prime = arc.all_conv_model(x, keep_prob)
else:
print("model requested not found, now some errors!")
# List of all Variables
variables = tf.all_variables()
# Load weights operator
print('save file is ./checkpoints/train_store_' + network_type)
ckpt = tf.train.get_checkpoint_state('./checkpoints/train_store_' + network_type)
weight_saver = tf.train.Saver(variables)
# Summary op
summary_op = tf.merge_all_summaries()
# Start running operations on the Graph.
sess = tf.Session()
# init if this is the very time training
weight_saver.restore(sess, ckpt.model_checkpoint_path)
print("restored from" + ckpt.model_checkpoint_path)
# Summary op
graph_def = sess.graph.as_graph_def(add_shapes=True)
dat = b.bounce_vec(32, FLAGS.num_balls, FLAGS.batch_size)
stddev_r = np.sum(sess.run([stddev],feed_dict={x:dat})[0], axis=0)/FLAGS.batch_size
sample_y = sess.run([y_sampled],feed_dict={x:dat})[0]
print(sample_y[0])
# create grid
y_p, x_p = np.mgrid[0:1:32j, 0:1:32j]
for i in xrange(10):
index = np.argmin(stddev_r)
for j in xrange(5):
z_f = np.copy(sample_y)
z_f[0,index] = 1.5*j - 3.0
print(sample_y[0])
print(z_f[0])
plt.subplot(10,5,j+5*i + 1)
plt.pcolor(x_p, y_p, sess.run([x_prime],feed_dict={y_sampled:z_f})[0][0,:,:,0])
if i == 9:
plt.xlabel(str(1.5*j - 3.0))
if j == 0:
plt.ylabel("{0:.2f}".format(stddev_r[index]))
plt.tick_params(axis='both', which='both', bottom='off', top='off', left='off', right='off', labelbottom='off', labelleft='off')
#plt.axis(frameon=False)
# just make it big to get out of the way
stddev_r[index] = 2.0
plt.savefig("figures/heat_map_" + network_type + ".png")
pylab.axis('off')
CreateMovie(filename, plotter, numframes, fps)
print 'making data...'
numballs = 1
numframes = 20
patchsize = 28
numcases = 70000
trainmovies = numpy.empty((numcases,numframes,patchsize**2), dtype=numpy.float32)
for i in range(numcases):
#print i
trainmovies[i] = bouncing_balls.bounce_vec(patchsize, numballs, numframes)
trainmovies = trainmovies.reshape(-1, numframes, patchsize**2)
np.save('/data/lisatmp2/kruegerd/bouncing_balls/bouncing_ball', trainmovies)
#trainmovies = numpy.load("100000bouncingballmovies16x16.npy")
#numtest = 10000
#testmovies = numpy.empty((numtest,numframes,patchsize**2), dtype=numpy.float32)
#for i in range(numtest):
#print i
# testmovies[i] = bouncing_balls.bounce_vec(patchsize, numballs, numframes)
#testmovies = testmovies.reshape(-1, numframes, patchsize**2)
print '... done'
def train(self,
valid_set=False,
learning_rate=0.1,
num_updates=500,
save=False,
output_folder=None,
lr_update=None,
mom_rate=0.9,
update_type='linear',
start=2,
batch_size=100,
filename=None):
self.best_train_cost = numpy.inf
self.best_valid_cost = numpy.inf
self.init_lr = learning_rate
self.lr = numpy.array(learning_rate)
self.mom_rate = mom_rate
self.output_folder = output_folder
self.valid_set = valid_set
self.save = save
self.lr_update = lr_update
self.stop_train = False
self.train_costs = []
self.valid_costs = []
self.num_updates = num_updates
self.batch_size = batch_size
self.update_type = update_type
self.start = start
self.filename = filename
self.valid_frequency = 1000
#self.save_model() #saving model after pre-training
try:
cost = []
for u in xrange(num_updates):
#pdb.set_trace()
if u % 10 == 0:
self.valid()
else:
batch_data = b.bounce_vec(
15, n=3,
T=128) #Ensure this is a list in the desired form.
fixed_array = numpy.zeros(batch_data.shape)
fixed_array[:] = batch_data
inputs = [fixed_array] + [self.lr]
if self.momentum:
inputs = inputs + [self.mom_rate]
no_update_cost = self.calc_cost(fixed_array)[0]
if numpy.isnan(no_update_cost):
print 'Training cost is NaN.'
print 'Breaking from training early, the last saved set of parameters is still usable!'
print 'Saving broken model for analysis.'
self.save_model('params_NaN.pickle')
print 'Saving input sequence'
path = os.path.join(self.output_folder, 'nan_seq.pkl')
pickle.dump(batch_data, open(path, 'w'))
break
else:
cost.append(self.f(*inputs))
mean_costs = numpy.mean(cost, axis=0)
print ' Update %i ' % (u + 1)
print '***Train Results***'
for i in xrange(self.num_costs):
print "Cost %i: %f" % (i, mean_costs[i])
self.train_costs.append(mean_costs)
this_cost = numpy.mean(cost, axis=0)[0]
if u > 0: #Because cost is not stable at the start of joint training of RNN_RNADE
if this_cost < self.best_train_cost:
self.best_train_cost = this_cost
print 'Best Params!'
if save:
self.save_model('best_params_train.pickle')
sys.stdout.flush()
if self.stop_train:
print 'Stopping training early.'
break
if lr_update:
self.update_lr(u + 1,
update_type=self.update_type,
start=self.start,
num_iterations=self.num_updates)
print 'Training completed!'
except KeyboardInterrupt:
print 'Training interrupted.'
def train(self,valid_set=False,learning_rate=0.1,num_updates=500,save=False,output_folder=None,lr_update=None,
mom_rate=0.9,update_type='linear',start=2,batch_size=100,filename=None):
self.best_train_cost = numpy.inf
self.best_valid_cost = numpy.inf
self.init_lr = learning_rate
self.lr = numpy.array(learning_rate)
self.mom_rate = mom_rate
self.output_folder = output_folder
self.valid_set = valid_set
self.save = save
self.lr_update = lr_update
self.stop_train = False
self.train_costs = []
self.valid_costs = []
self.num_updates = num_updates
self.batch_size = batch_size
self.update_type = update_type
self.start = start
self.filename = filename
self.valid_frequency = 1000
#self.save_model() #saving model after pre-training
try:
cost = []
for u in xrange(num_updates):
#pdb.set_trace()
if u%10 == 0:
self.valid()
else:
batch_data = b.bounce_vec(15,n=3,T=128) #Ensure this is a list in the desired form.
fixed_array = numpy.zeros(batch_data.shape)
fixed_array[:] = batch_data
inputs = [fixed_array] + [self.lr]
if self.momentum:
inputs = inputs + [self.mom_rate]
no_update_cost = self.calc_cost(fixed_array)[0]
if numpy.isnan(no_update_cost):
print 'Training cost is NaN.'
print 'Breaking from training early, the last saved set of parameters is still usable!'
print 'Saving broken model for analysis.'
self.save_model('params_NaN.pickle')
print 'Saving input sequence'
path = os.path.join(self.output_folder,'nan_seq.pkl')
pickle.dump(batch_data,open(path,'w'))
break
else:
cost.append(self.f(*inputs))
mean_costs = numpy.mean(cost,axis=0)
print ' Update %i ' %(u+1)
print '***Train Results***'
for i in xrange(self.num_costs):
print "Cost %i: %f"%(i,mean_costs[i])
self.train_costs.append(mean_costs)
this_cost = numpy.mean(cost, axis=0)[0]
if u > 0: #Because cost is not stable at the start of joint training of RNN_RNADE
if this_cost < self.best_train_cost:
self.best_train_cost = this_cost
print 'Best Params!'
if save:
self.save_model('best_params_train.pickle')
sys.stdout.flush()
if self.stop_train:
print 'Stopping training early.'
break
if lr_update:
self.update_lr(u+1,update_type=self.update_type,start=self.start,num_iterations=self.num_updates)
print 'Training completed!'
except KeyboardInterrupt:
print 'Training interrupted.'
