def train(self,
x,
y,
x_valid,
y_valid,
epochs,
num_batches,
print_every=1,
learning_rate=3e-4,
beta1=0.9,
beta2=0.999,
seed=31415,
stop_iter=100,
save_path=None,
load_path=None):
self.num_examples = x.shape[0]
self.num_batches = num_batches
assert self.num_examples % self.num_batches == 0, '#Examples % #Batches != 0'
self.batch_size = self.num_examples // self.num_batches
''' Session and Summary '''
self.save_path = save_path
if save_path is None:
self.save_ckpt_path = 'checkpoints/model_VAE_{}-{}_{}.cpkt'.format(
learning_rate, self.batch_size, time.time())
else:
self.save_ckpt_path = save_path + 'model_VAE_{}-{}_{}.cpkt'.format(
learning_rate, self.batch_size, time.time())
np.random.seed(seed)
tf.set_random_seed(seed)
with self.G.as_default():
self.optimizer_origin = tf.train.AdamOptimizer(
learning_rate=learning_rate, beta1=beta1, beta2=beta2)
self.optimizer = tf.contrib.estimator.clip_gradients_by_norm(
self.optimizer_origin, clip_norm=1.0)
self.train_op = self.optimizer.minimize(self.cost)
init = tf.global_variables_initializer()
self._test_vars = None
with self.session as sess:
sess.run(init)
if load_path == 'default':
full_path = tf.train.latest_checkpoint(self.save_path)
print('restore model from {}'.format(full_path))
self.saver.restore(sess, full_path)
elif load_path is not None:
full_path = tf.train.latest_checkpoint(load_path)
print('restore model from {}'.format(full_path))
self.saver.restore(sess, full_path)
training_cost = 0.
best_eval_log_lik = -np.inf
best_mse = np.inf
stop_counter = 0
for epoch in range(epochs):
# TODO create shuffle Data
''' Training '''
for x_batch, y_batch in utils.feed_numpy(
self.batch_size, x, y):
training_result = sess.run([self.train_op, self.cost],
feed_dict={
self.x: x_batch,
self.y: y_batch
})
training_cost = training_result[1]
''' Evaluation '''
stop_counter += 1
if epoch % print_every == 0:
test_vars = tf.get_collection(
bookkeeper.GraphKeys.TEST_VARIABLES)
if test_vars:
if test_vars != self._test_vars:
self._test_vars = list(test_vars)
self._test_var_init_op = tf.initialize_variables(
test_vars)
self._test_var_init_op.run()
mse = sess.run(self.mse_loss,
feed_dict={
self.x: x,
self.y: y
})
# corr_loss = self.corr_loss.eval(feed_dict={self.x: x, self.y: y})
if mse < best_mse:
best_eval_log_lik = mse
self.saver.save(sess, self.save_ckpt_path)
stop_counter = 0
utils.print_metrics(
epoch + 1,
['Training', 'cost', training_cost],
# ['AutoCorr', 'train', corr_loss],
['MSE ', 'train', mse])
if stop_counter >= stop_iter:
print(
'Stopping No change in validation log-likelihood for {} iterations'
.format(stop_iter))
print('Best validation log-likelihood: {}'.format(
best_eval_log_lik))
print('Model saved in {}'.format(self.save_path))
break
def train( self, x, x_valid,
epochs, num_batches,
print_every = 1,
learning_rate = 3e-4,
beta1 = 0.9,
beta2 = 0.999,
seed = 31415,
stop_iter = 100,
save_path = None,
load_path = None,
draw_img = 1 ):
self.num_examples = x.shape[0]
self.num_batches = num_batches
assert self.num_examples % self.num_batches == 0, '#Examples % #Batches != 0'
self.batch_size = self.num_examples // self.num_batches
''' Session and Summary '''
if save_path is None:
self.save_path = 'checkpoints/model_VAE_{}-{}_{}.cpkt'.format(learning_rate,self.batch_size,time.time())
else:
self.save_path = save_path
np.random.seed(seed)
tf.set_random_seed(seed)
with self.G.as_default():
self.optimiser = tf.train.AdamOptimizer( learning_rate = learning_rate, beta1 = beta1, beta2 = beta2 )
self.train_op = self.optimiser.minimize( self.cost )
init = tf.initialize_all_variables()
self._test_vars = None
with self.session as sess:
sess.run(init)
if load_path == 'default': self.saver.restore( sess, self.save_path )
elif load_path is not None: self.saver.restore( sess, load_path )
training_cost = 0.
best_eval_log_lik = - np.inf
stop_counter = 0
for epoch in range(epochs):
''' Shuffle Data '''
np.random.shuffle( x )
''' Training '''
for x_batch in utils.feed_numpy( self.batch_size, x ):
training_result = sess.run( [self.train_op, self.cost],
feed_dict = { self.x: x_batch } )
training_cost = training_result[1]
''' Evaluation '''
stop_counter += 1
if epoch % print_every == 0:
test_vars = tf.get_collection(bookkeeper.GraphKeys.TEST_VARIABLES)
if test_vars:
if test_vars != self._test_vars:
self._test_vars = list(test_vars)
self._test_var_init_op = tf.initialize_variables(test_vars)
self._test_var_init_op.run()
eval_log_lik, x_recon_eval = \
sess.run( [self.eval_log_lik, self.x_recon_eval],
feed_dict = { self.x: x_valid } )
if eval_log_lik > best_eval_log_lik:
best_eval_log_lik = eval_log_lik
self.saver.save( sess, self.save_path )
stop_counter = 0
utils.print_metrics( epoch+1,
['Training', 'cost', training_cost],
['Validation', 'log-likelihood', eval_log_lik] )
if draw_img > 0 and epoch % draw_img == 0:
import matplotlib
matplotlib.use('Agg')
import pylab
import seaborn as sns
five_random = np.random.random_integers(x_valid.shape[0], size = 5)
x_sample = x_valid[five_random]
x_recon_sample = x_recon_eval[five_random]
sns.set_style('white')
f, axes = pylab.subplots(5, 2, figsize=(8,12))
for i,row in enumerate(axes):
row[0].imshow(x_sample[i].reshape(28, 28), vmin=0, vmax=1)
im = row[1].imshow(x_recon_sample[i].reshape(28, 28), vmin=0, vmax=1,
cmap=sns.light_palette((1.0, 0.4980, 0.0549), input="rgb", as_cmap=True))
pylab.setp([a.get_xticklabels() for a in row], visible=False)
pylab.setp([a.get_yticklabels() for a in row], visible=False)
f.subplots_adjust(left=0.0, right=0.9, bottom=0.0, top=1.0)
cbar_ax = f.add_axes([0.9, 0.1, 0.04, 0.8])
f.colorbar(im, cax=cbar_ax, use_gridspec=True)
pylab.tight_layout()
pylab.savefig('img/recon-'+str(epoch)+'.png', format='png')
pylab.clf()
pylab.close('all')
if stop_counter >= stop_iter:
print('Stopping VAE training')
print('No change in validation log-likelihood for {} iterations'.format(stop_iter))
print('Best validation log-likelihood: {}'.format(best_eval_log_lik))
print('Model saved in {}'.format(self.save_path))
break
def train(self,
x,
x_valid,
epochs,
num_batches,
print_every=1,
learning_rate=3e-4,
beta1=0.9,
beta2=0.999,
seed=31415,
stop_iter=100,
save_path=None,
load_path=None,
draw_img=1):
self.num_examples = x.shape[0]
self.num_batches = num_batches
print("Num examples: {}".format(self.num_examples))
assert self.num_examples % self.num_batches == 0, '#Examples % #Batches != 0'
self.batch_size = self.num_examples // self.num_batches
''' Session and Summary '''
if save_path is None:
self.save_path = 'checkpoints/model_VAE_{}-{}_{}.cpkt'.format(
learning_rate, self.batch_size, time.time())
else:
self.save_path = save_path
np.random.seed(seed)
tf.set_random_seed(seed)
with self.G.as_default():
self.optimiser = tf.train.AdamOptimizer(
learning_rate=learning_rate, beta1=beta1, beta2=beta2)
self.train_op = self.optimiser.minimize(self.cost)
init = tf.global_variables_initializer()
self._test_vars = None
with self.session as sess:
sess.run(init)
if load_path == 'default': self.saver.restore(sess, self.save_path)
elif load_path is not None: self.saver.restore(sess, load_path)
training_cost = 0.
best_eval_log_lik = -np.inf
stop_counter = 0
for epoch in range(epochs):
''' Shuffle Data '''
np.random.shuffle(x)
''' Training '''
for x_batch in utils.feed_numpy(self.batch_size, x):
training_result = sess.run([self.train_op, self.cost],
feed_dict={self.x: x_batch})
training_cost = training_result[1]
''' Evaluation '''
stop_counter += 1
if epoch % print_every == 0:
test_vars = tf.get_collection(
bookkeeper.GraphKeys.TEST_VARIABLES)
if test_vars:
if test_vars != self._test_vars:
self._test_vars = list(test_vars)
self._test_var_init_op = tf.initialize_variables(
test_vars)
self._test_var_init_op.run()
eval_log_lik, x_recon_eval = \
sess.run( [self.eval_log_lik, self.x_recon_eval],
feed_dict = { self.x: x_valid } )
if eval_log_lik > best_eval_log_lik:
print('Saving')
best_eval_log_lik = eval_log_lik
self.saver.save(sess, self.save_path)
stop_counter = 0
utils.print_metrics(
epoch + 1, ['Training', 'cost', training_cost],
['Validation', 'log-likelihood', eval_log_lik])
if draw_img > 0 and epoch % draw_img == 0:
import matplotlib
matplotlib.use('Agg')
import pylab
import seaborn as sns
five_random = np.random.random_integers(
x_valid.shape[0], size=5)
x_sample = x_valid[five_random]
x_recon_sample = x_recon_eval[five_random]
sns.set_style('white')
f, axes = pylab.subplots(5, 2, figsize=(8, 12))
for i, row in enumerate(axes):
row[0].imshow(x_sample[i].reshape(28, 28),
vmin=0,
vmax=1)
im = row[1].imshow(x_recon_sample[i].reshape(
28, 28),
vmin=0,
vmax=1,
cmap=sns.light_palette(
(1.0, 0.4980, 0.0549),
input="rgb",
as_cmap=True))
pylab.setp([a.get_xticklabels() for a in row],
visible=False)
pylab.setp([a.get_yticklabels() for a in row],
visible=False)
f.subplots_adjust(left=0.0,
right=0.9,
bottom=0.0,
top=1.0)
cbar_ax = f.add_axes([0.9, 0.1, 0.04, 0.8])
f.colorbar(im, cax=cbar_ax, use_gridspec=True)
pylab.tight_layout()
pylab.savefig('img/recon-' + str(epoch) + '.png',
format='png')
pylab.clf()
pylab.close('all')
if stop_counter >= stop_iter:
print('Stopping VAE training')
print(
'No change in validation log-likelihood for {} iterations'
.format(stop_iter))
print('Best validation log-likelihood: {}'.format(
best_eval_log_lik))
print('Model saved in {}'.format(self.save_path))
break
def train(self, x, x_con, x_valid, x_con_valid, params,
y_one_hot_train=None,
y_one_hot_valid=None,
save_path=None,
load_path=None
):
stop_iter = self.params.stop_iter
N, M = params.n_batch, params.batch_size
lr, seed, epochs, print_every = params.learning_rate, params.seed, params.epochs, params.print_every
''' Session and Summary '''
self.save_path = save_path
if save_path is None:
self.save_ckpt_path = 'checkpoints/model_VAE_{}-{}_{}.cpkt'.format(lr, M, time.time())
else:
self.save_ckpt_path = save_path + 'model_VAE_{}-{}_{}.cpkt'.format(lr, M, time.time())
np.random.seed(seed)
tf.set_random_seed(seed)
with self.G.as_default():
self.optimizer_origin = tf.train.AdamOptimizer(learning_rate=lr)
self.optimizer = tf.contrib.estimator.clip_gradients_by_norm(self.optimizer_origin, clip_norm=1.0)
self.train_op = self.optimizer.minimize(self.cost)
self.init_g = tf.global_variables_initializer()
self.init_l = tf.local_variables_initializer()
self._test_vars = None
with self.session as sess:
sess.run(self.init_g)
sess.run(self.init_l)
if load_path == 'default':
full_path = tf.train.latest_checkpoint(self.save_path)
print('restore model from {}'.format(full_path))
self.saver.restore(sess, full_path)
elif load_path is not None:
full_path = tf.train.latest_checkpoint(load_path)
print('restore model from {}'.format(full_path))
self.saver.restore(sess, full_path)
best_eval_log_lik = - np.inf
best_mse = np.inf
stop_counter = 0
for epoch in range(epochs):
# TODO create shuffle Data
# X_train_shuffle, X_tau_train_shuffle, y_classify_train_shuffle = shuffle_data(X_train, X_tau_train, y_classify_train)
''' Training '''
training_cost, accuracy, mse = 0, 0, 0
for x_batch, x_con_batch, y_one_hot_batch in utils.feed_numpy(x, x_con, batch_size=M,
y=y_one_hot_train):
training_result = sess.run([self.train_op, self.cost, self.accuracy, self.mse_loss],
feed_dict={self.x: x_batch, self.x_con: x_con_batch,
self.y_one_hot: y_one_hot_batch})
training_cost += training_result[1]
accuracy += training_result[2]
mse += training_result[3]
training_cost, accuracy, mse = training_cost / N, accuracy / N, mse / N
''' Evaluation '''
stop_counter += 1
if epoch % print_every == 0:
# mse = sess.run(self.mse_loss, feed_dict={self.x: x, self.x_con: x_con})
val_result = sess.run(self.accuracy,
feed_dict={self.x: x_valid, self.x_con: x_con_valid,
self.y_one_hot: y_one_hot_valid})
if mse < best_mse:
best_eval_log_lik = mse
self.saver.save(sess, self.save_ckpt_path)
stop_counter = 0
utils.print_metrics(epoch + 1,
['Training cost', training_cost],
['Accuracy', accuracy],
['MSE', mse],
['Validation accuracy', val_result]
)
if stop_counter >= stop_iter:
print('Stopping No change in validation log-likelihood for {} iterations'.format(stop_iter))
print('Best validation log-likelihood: {}'.format(best_eval_log_lik))
print('Model saved in {}'.format(self.save_path))
break