def pretrain(self, sess, x, X_train, X_val, denoise_name=None, n_epoch=100, batch_size=128, print_freq=10,
save=True, save_name='w1pre_'):
''' pre-train the parameters of previous DenseLayer '''
print(" tensorlayer: %s start pretrain" % self.name)
print(" batch_size: %d" % batch_size)
if denoise_name:
print(" denoising layer keep: %f" % self.all_drop[set_keep[denoise_name]])
dp_denoise = self.all_drop[set_keep[denoise_name]]
else:
print(" no denoising layer")
# You may want to check different cost values
# dp_dict = utils.dict_to_one( self.all_drop )
# feed_dict = {x: X_val}
# feed_dict.update(dp_dict)
# print(sess.run([self.mse, self.L1_a, self.L2_w], feed_dict=feed_dict))
# exit()
for epoch in range(n_epoch):
start_time = time.time()
for X_train_a, _ in iterate.minibatches(X_train, X_train, batch_size, shuffle=True):
dp_dict = utils.dict_to_one( self.all_drop )
if denoise_name:
dp_dict[set_keep[denoise_name]] = dp_denoise
feed_dict = {x: X_train_a}
feed_dict.update(dp_dict)
sess.run(self.train_op, feed_dict=feed_dict)
if epoch + 1 == 1 or (epoch + 1) % print_freq == 0:
print("Epoch %d of %d took %fs" % (epoch + 1, n_epoch, time.time() - start_time))
train_loss, n_batch = 0, 0
for X_train_a, _ in iterate.minibatches(X_train, X_train, batch_size, shuffle=True):
dp_dict = utils.dict_to_one( self.all_drop )
feed_dict = {x: X_train_a}
feed_dict.update(dp_dict)
err = sess.run(self.cost, feed_dict=feed_dict)
train_loss += err
n_batch += 1
print(" train loss: %f" % (train_loss/ n_batch))
val_loss, n_batch = 0, 0
for X_val_a, _ in iterate.minibatches(X_val, X_val, batch_size, shuffle=True):
dp_dict = utils.dict_to_one( self.all_drop )
feed_dict = {x: X_val_a}
feed_dict.update(dp_dict)
err = sess.run(self.cost, feed_dict=feed_dict)
val_loss += err
n_batch += 1
print(" val loss: %f" % (val_loss/ n_batch))
if save:
try:
visualize.W(self.train_params[0].eval(), second=10, saveable=True, name=save_name+str(epoch+1), fig_idx=2012)
except:
raise Exception("You should change visualize.W(), if you want to save the feature images for different dataset")
def pretrain(self, sess, x, X_train, X_val, denoise_name=None, n_epoch=100, batch_size=128, print_freq=10,
save=True, save_name='w1pre_'):
# ====================================================
#
# You need to modify the cost function in __init__() so as to
# get your own pre-train method.
#
# ====================================================
print(" tensorlayer: %s start pretrain" % self.name)
print(" batch_size: %d" % batch_size)
if denoise_name:
print(" denoising layer keep: %f" % self.all_drop[set_keep[denoise_name]])
dp_denoise = self.all_drop[set_keep[denoise_name]]
else:
print(" no denoising layer")
for epoch in range(n_epoch):
start_time = time.time()
for X_train_a, _ in iterate.minibatches(X_train, X_train, batch_size, shuffle=True):
dp_dict = utils.dict_to_one( self.all_drop )
if denoise_name:
dp_dict[set_keep[denoise_name]] = dp_denoise
feed_dict = {x: X_train_a}
feed_dict.update(dp_dict)
sess.run(self.train_op, feed_dict=feed_dict)
if epoch + 1 == 1 or (epoch + 1) % print_freq == 0:
print("Epoch %d of %d took %fs" % (epoch + 1, n_epoch, time.time() - start_time))
train_loss, n_batch = 0, 0
for X_train_a, _ in iterate.minibatches(X_train, X_train, batch_size, shuffle=True):
dp_dict = utils.dict_to_one( self.all_drop )
feed_dict = {x: X_train_a}
feed_dict.update(dp_dict)
err = sess.run(self.cost, feed_dict=feed_dict)
train_loss += err
n_batch += 1
print(" train loss: %f" % (train_loss/ n_batch))
val_loss, n_batch = 0, 0
for X_val_a, _ in iterate.minibatches(X_val, X_val, batch_size, shuffle=True):
dp_dict = utils.dict_to_one( self.all_drop )
feed_dict = {x: X_val_a}
feed_dict.update(dp_dict)
err = sess.run(self.cost, feed_dict=feed_dict)
val_loss += err
n_batch += 1
print(" val loss: %f" % (val_loss/ n_batch))
if save:
try:
visualize.W(self.train_params[0].eval(), second=10, saveable=True, shape=[28,28], name=save_name+str(epoch+1), fig_idx=2012)
files.save_npz([self.all_params[0]] , name=save_name+str(epoch+1)+'.npz')
except:
raise Exception("You should change visualize.W(), if you want to save the feature images for different dataset")
def test(sess, network, acc, X_test, y_test, x, y_, batch_size, cost=None):
"""
Test a given non time-series network by the given test data and metric.
Parameters
----------
sess : Session
TensorFlow session.
network : TensorLayer layer
The network.
acc : TensorFlow expression or None
Metric for accuracy or others.
- If None, would not print the information.
X_test : numpy.array
The input of testing data.
y_test : numpy array
The target of testing data
x : placeholder
For inputs.
y_ : placeholder
For targets.
batch_size : int or None
The batch size for testing, when dataset is large, we should use minibatche for testing;
if dataset is small, we can set it to None.
cost : TensorFlow expression or None
Metric for cost or others. If None, would not print the information.
Examples
--------
See `tutorial_mnist_simple.py <https://github.com/tensorlayer/tensorlayer/blob/master/example/tutorial_mnist_simple.py>`_
>>> tl.utils.test(sess, network, acc, X_test, y_test, x, y_, batch_size=None, cost=cost)
"""
logging.info('Start testing the network ...')
if batch_size is None:
dp_dict = dict_to_one(network.all_drop)
feed_dict = {x: X_test, y_: y_test}
feed_dict.update(dp_dict)
if cost is not None:
logging.info(" test loss: %f" % sess.run(cost, feed_dict=feed_dict))
logging.info(" test acc: %f" % sess.run(acc, feed_dict=feed_dict))
# logging.info(" test acc: %f" % np.mean(y_test == sess.run(y_op,
# feed_dict=feed_dict)))
else:
test_loss, test_acc, n_batch = 0, 0, 0
for X_test_a, y_test_a in iterate.minibatches(X_test, y_test, batch_size, shuffle=True):
dp_dict = dict_to_one(network.all_drop) # disable noise layers
feed_dict = {x: X_test_a, y_: y_test_a}
feed_dict.update(dp_dict)
if cost is not None:
err, ac = sess.run([cost, acc], feed_dict=feed_dict)
test_loss += err
else:
ac = sess.run(acc, feed_dict=feed_dict)
test_acc += ac
n_batch += 1
if cost is not None:
logging.info(" test loss: %f" % (test_loss / n_batch))
logging.info(" test acc: %f" % (test_acc / n_batch))
def save_images(self, nets, epoch, curr_tr, images_i, images_j):
"""
Saves input and output images.
"""
nets = utils.set_mode(nets, "eval")
exists_or_mkdir(self._images_dir)
if curr_tr > 0:
donorm = False
else:
donorm = True
names = [
'inputA_', 'inputB_', 'fakeA_', 'fakeB_', 'cycA_', 'cycB_',
'mask_a', 'mask_b'
]
with open(
os.path.join(self._output_dir,
'epoch_' + str(epoch) + '.html'), 'w') as v_html:
input_iter = minibatches(images_i,
images_j,
batch_size=1,
shuffle=True)
for i in range(0, self._num_imgs_to_save):
#pdb.set_trace()
print("Saving image {}/{}...".format(i,
self._num_imgs_to_save))
self.image_a, self.image_b = next(input_iter)
tmp_imgA = self.get_fake_image_pool(self.num_fake_inputs,
self.fake_images_A)
self.fake_pool_A_mask = tmp_imgA["mask"]
self.fake_pool_A = tmp_imgA["im"]
tmp_imgB = self.get_fake_image_pool(self.num_fake_inputs,
self.fake_images_B)
self.fake_pool_B_mask = tmp_imgB["mask"]
self.fake_pool_B = tmp_imgB["im"]
self.transition_rate = np.array([curr_tr], dtype=np.float32)
self.donorm = np.array([donorm], dtype=np.float32)
self.output_converter(
model.get_outputs(self.input_converter(), nets))
figures_to_save = [
self.image_a, self.image_b, self.fake_images_b,
self.fake_images_a, self.cycle_images_a,
self.cycle_images_b, self.masks[0], self.masks[1]
]
self.figure_writer(figures_to_save,
names,
v_html,
epoch,
i,
html_mode=0)
def pretrain(self,
sess,
x,
X_train,
X_val,
denoise_name=None,
n_epoch=100,
batch_size=128,
print_freq=10,
save=True,
save_name='w1pre_'):
# ====================================================
#
# You need to modify the cost function in __init__() so as to
# get your own pre-train method.
#
# ====================================================
logging.info(" [*] %s start pretrain" % self.name)
logging.info(" batch_size: %d" % batch_size)
if denoise_name:
logging.info(" denoising layer keep: %f" %
self.all_drop[LayersConfig.set_keep[denoise_name]])
dp_denoise = self.all_drop[LayersConfig.set_keep[denoise_name]]
else:
logging.info(" no denoising layer")
for epoch in range(n_epoch):
start_time = time.time()
for X_train_a, _ in iterate.minibatches(X_train,
X_train,
batch_size,
shuffle=True):
dp_dict = utils.dict_to_one(self.all_drop)
if denoise_name:
dp_dict[LayersConfig.set_keep[denoise_name]] = dp_denoise
feed_dict = {x: X_train_a}
feed_dict.update(dp_dict)
sess.run(self.train_op, feed_dict=feed_dict)
if epoch + 1 == 1 or (epoch + 1) % print_freq == 0:
logging.info("Epoch %d of %d took %fs" %
(epoch + 1, n_epoch, time.time() - start_time))
train_loss, n_batch = 0, 0
for X_train_a, _ in iterate.minibatches(X_train,
X_train,
batch_size,
shuffle=True):
dp_dict = utils.dict_to_one(self.all_drop)
feed_dict = {x: X_train_a}
feed_dict.update(dp_dict)
err = sess.run(self.cost, feed_dict=feed_dict)
train_loss += err
n_batch += 1
logging.info(" train loss: %f" % (train_loss / n_batch))
val_loss, n_batch = 0, 0
for X_val_a, _ in iterate.minibatches(X_val,
X_val,
batch_size,
shuffle=True):
dp_dict = utils.dict_to_one(self.all_drop)
feed_dict = {x: X_val_a}
feed_dict.update(dp_dict)
err = sess.run(self.cost, feed_dict=feed_dict)
val_loss += err
n_batch += 1
logging.info(" val loss: %f" % (val_loss / n_batch))
if save:
try:
visualize.draw_weights(self.train_params[0].eval(),
second=10,
saveable=True,
shape=[28, 28],
name=save_name + str(epoch + 1),
fig_idx=2012)
files.save_npz([self.all_params[0]],
name=save_name + str(epoch + 1) +
'.npz')
except Exception:
raise Exception(
"You should change the visualize.W() in ReconLayer.pretrain(), if you want to save the feature images for different dataset"
)
def train(self):
"""
Training Function.
We use batch size = 1 for training
"""
# Build the network and compute losses
nets = self.model_setup()
summary_writer = tf.summary.create_file_writer(
os.path.join(self._output_dir, "log"))
summary_writer.set_as_default()
max_images = cyclegan_datasets.DATASET_TO_SIZES[self._dataset_name]
half_training_ep = int(self._max_step / 2)
# Restore the model to run the model from last checkpoint
print("Loading the latest checkpoint...")
if self._to_restore:
checkpoint_name = os.path.join(self._checkpoint_dir,
self._checkpoint_name)
utils.load(checkpoint_name, nets=nets)
self.global_step = int(checkpoint_name[-2:])
else:
self.global_step = 0
exists_or_mkdir(self._output_dir)
self.upd_fake_image_pool(self.num_fake_inputs, self.fake_pool_A,
self.fake_pool_A_mask, self.fake_images_A)
self.upd_fake_image_pool(self.num_fake_inputs, self.fake_pool_B,
self.fake_pool_B_mask, self.fake_images_B)
self.num_fake_inputs += 1
# Training Loop
for epoch in range(self.global_step, self._max_step):
print("In the epoch ", epoch)
print("Saving the latest checkpoint...")
utils.save(nets,
os.path.join(self._output_dir, "AGGAN_%02d" % epoch))
# Setting lr
curr_lr = self._base_lr
if epoch >= half_training_ep:
curr_lr -= self._base_lr * (
epoch - half_training_ep) / half_training_ep
self.g_A_trainer.learning_rate = curr_lr
self.g_B_trainer.learning_rate = curr_lr
self.g_A_trainer_bis.learning_rate = curr_lr
self.g_B_trainer_bis.learning_rate = curr_lr
self.d_A_trainer.learning_rate = curr_lr
self.d_B_trainer.learning_rate = curr_lr
if epoch < self._switch:
curr_tr = 0
donorm = True
to_train_A = self.g_A_trainer
to_train_B = self.g_B_trainer
to_train_A_vars = self.g_A_vars + self.g_Ae_vars
to_train_B_vars = self.g_B_vars + self.g_Be_vars
else:
curr_tr = self._threshold_fg
donorm = False
to_train_A = self.g_A_trainer_bis
to_train_B = self.g_B_trainer_bis
to_train_A_vars = self.g_A_vars
to_train_B_vars = self.g_B_vars
print("Loading data...")
tot_inputs = data_loader.load_data(self._dataset_name,
self._size_before_crop, False,
self._do_flipping)
self.inputs_img_i = tot_inputs['images_i']
self.inputs_img_j = tot_inputs['images_j']
assert (len(self.inputs_img_i) == len(self.inputs_img_j)
and max_images == len(self.inputs_img_i))
self.save_images(nets, epoch, curr_tr, self.inputs_img_i,
self.inputs_img_j)
nets = utils.set_mode(nets, "train")
input_iter = minibatches(self.inputs_img_i,
self.inputs_img_j,
batch_size=1,
shuffle=True)
for i in range(max_images):
print("Processing batch {}/{} in {}th epoch".format(
i, max_images, epoch))
self.image_a, self.image_b = next(input_iter)
tmp_imgA = self.get_fake_image_pool(self.num_fake_inputs,
self.fake_images_A)
self.fake_pool_A_mask = tmp_imgA["mask"]
self.fake_pool_A = tmp_imgA["im"]
tmp_imgB = self.get_fake_image_pool(self.num_fake_inputs,
self.fake_images_B)
self.fake_pool_B_mask = tmp_imgB["mask"]
self.fake_pool_B = tmp_imgB["im"]
self.transition_rate = np.array([curr_tr], dtype=np.float32)
self.donorm = np.array([donorm], dtype=np.float32)
with tf.GradientTape(persistent=True) as tape:
self.output_converter(
model.get_outputs(self.input_converter(), nets))
self.upd_fake_image_pool(self.num_fake_inputs,
self.fake_images_b, self.masks[0],
self.fake_images_B)
self.upd_fake_image_pool(self.num_fake_inputs,
self.fake_images_a, self.masks[1],
self.fake_images_A)
self.compute_losses()
#pdb.set_trace()
grad = tape.gradient(self.d_B_loss, self.d_B_vars)
self.d_B_trainer.apply_gradients(zip(grad, self.d_B_vars))
grad = tape.gradient(self.d_A_loss, self.d_A_vars)
self.d_A_trainer.apply_gradients(zip(grad, self.d_A_vars))
grad = tape.gradient(self.g_A_loss, to_train_A_vars)
to_train_A.apply_gradients(zip(grad, to_train_A_vars))
grad = tape.gradient(self.g_B_loss, to_train_B_vars)
to_train_B.apply_gradients(zip(grad, to_train_B_vars))
tot_loss = self.g_A_loss + self.g_B_loss + self.d_A_loss + self.d_B_loss
print("[training_info] g_A_loss = {}, g_B_loss = {}, d_A_loss = {}, d_B_loss = {}, \
tot_loss = {}, lr={}, curr_tr={}" .format(self.g_A_loss, self.g_B_loss, self.d_A_loss, \
self.d_B_loss, tot_loss, curr_lr, curr_tr))
tf.summary.scalar('g_A_loss',
self.g_A_loss,
step=self.global_step * max_images + i)
tf.summary.scalar('g_B_loss',
self.g_B_loss,
step=self.global_step * max_images + i)
tf.summary.scalar('d_A_loss',
self.d_A_loss,
step=self.global_step * max_images + i)
tf.summary.scalar('d_B_loss',
self.d_B_loss,
step=self.global_step * max_images + i)
tf.summary.scalar('learning_rate',
to_train_A.learning_rate,
step=self.global_step * max_images + i)
tf.summary.scalar('total_loss',
tot_loss,
step=self.global_step * max_images + i)
self.num_fake_inputs += 1
self.global_step = epoch + 1
summary_writer.flush()
def save_images_bis(self, nets, epoch, images_i, images_j):
"""
Saves input and output images.
"""
names = [
'input_A_', 'mask_A_', 'masked_inputA_', 'fakeB_', 'input_B_',
'mask_B_', 'masked_inputB_', 'fakeA_'
]
space = '                        ' \
'                        ' \
'         '
nets = utils.set_mode(nets, "eval")
#pdb.set_trace()
exists_or_mkdir(self._images_dir)
with open(
os.path.join(self._output_dir,
'results_' + str(epoch) + '.html'),
'w') as v_html:
v_html.write("<b>Inputs" + space + "Masks" + space +
"Masked_images" + space + "Generated_images</b>")
v_html.write("<br>")
input_iter = minibatches(images_i,
images_j,
batch_size=1,
shuffle=True)
for i in range(0, self._num_imgs_to_save):
print("Saving image {}/{}...".format(i,
self._num_imgs_to_save))
#pdb.set_trace()
self.image_a, self.image_b = next(input_iter)
tmp_imgA = self.get_fake_image_pool(self.num_fake_inputs,
self.fake_images_A)
self.fake_pool_A_mask = tmp_imgA["mask"]
self.fake_pool_A = tmp_imgA["im"]
tmp_imgB = self.get_fake_image_pool(self.num_fake_inputs,
self.fake_images_B)
self.fake_pool_B_mask = tmp_imgB["mask"]
self.fake_pool_B = tmp_imgB["im"]
self.transition_rate = np.array([0.1], dtype=np.float32)
self.donorm = np.array([True], dtype=np.float32)
self.output_converter(
model.get_outputs(self.input_converter(), nets))
figures_to_save = [
self.image_a, self.masks[0], self.masked_ims[0],
self.fake_images_b, self.image_b, self.masks[1],
self.masked_ims[1], self.fake_images_a
]
self.figure_writer(figures_to_save,
names,
v_html,
epoch,
i,
html_mode=1)
sess.run(init_op)
# Run training loop
for layer in x_model.layers:
layer.trainable = False
with sess.as_default():
#saver = tf.train.Saver()
if goon:
#saver.restore(sess,'./model.ckpt')
x_model.load_weights('x_model.h5')
print('OK load!!!!!!!!!!')
else:
print('Creat new!!!!!!!!!!')
for i in range(100):
for X_train_a, y_train_a in minibatches(X_train,
y_train,
batch_size,
shuffle=True):
_, _ = sess.run([cost, train_op],
feed_dict={
img: X_train_a,
y_: y_train_a,
K.learning_phase(): 1
})
train_loss, train_acc, n_batch = 0, 0, 0
for X_train_a, y_train_a in minibatches(X_train,
y_train,
batch_size,
shuffle=False):
err, ac = sess.run([cost, acc],
feed_dict={
def pretrain(self,
sess,
x,
X_train,
X_val,
denoise_name=None,
n_epoch=100,
batch_size=128,
print_freq=10,
save=True,
save_name='w1pre_'):
''' pre-train the parameters of previous DenseLayer '''
print(" tensorlayer: %s start pretrain" % self.name)
print(" batch_size: %d" % batch_size)
if denoise_name:
print(" denoising layer keep: %f" %
self.all_drop[set_keep[denoise_name]])
dp_denoise = self.all_drop[set_keep[denoise_name]]
else:
print(" no denoising layer")
# You may want to check different cost values
# dp_dict = utils.dict_to_one( self.all_drop )
# feed_dict = {x: X_val}
# feed_dict.update(dp_dict)
# print(sess.run([self.mse, self.L1_a, self.L2_w], feed_dict=feed_dict))
# exit()
for epoch in range(n_epoch):
start_time = time.time()
for X_train_a, _ in iterate.minibatches(X_train,
X_train,
batch_size,
shuffle=True):
dp_dict = utils.dict_to_one(self.all_drop)
if denoise_name:
dp_dict[set_keep[denoise_name]] = dp_denoise
feed_dict = {x: X_train_a}
feed_dict.update(dp_dict)
sess.run(self.train_op, feed_dict=feed_dict)
if epoch + 1 == 1 or (epoch + 1) % print_freq == 0:
print("Epoch %d of %d took %fs" %
(epoch + 1, n_epoch, time.time() - start_time))
train_loss, n_batch = 0, 0
for X_train_a, _ in iterate.minibatches(X_train,
X_train,
batch_size,
shuffle=True):
dp_dict = utils.dict_to_one(self.all_drop)
feed_dict = {x: X_train_a}
feed_dict.update(dp_dict)
err = sess.run(self.cost, feed_dict=feed_dict)
train_loss += err
n_batch += 1
print(" train loss: %f" % (train_loss / n_batch))
val_loss, n_batch = 0, 0
for X_val_a, _ in iterate.minibatches(X_val,
X_val,
batch_size,
shuffle=True):
dp_dict = utils.dict_to_one(self.all_drop)
feed_dict = {x: X_val_a}
feed_dict.update(dp_dict)
err = sess.run(self.cost, feed_dict=feed_dict)
val_loss += err
n_batch += 1
print(" val loss: %f" % (val_loss / n_batch))
if save:
try:
visualize.W(self.train_params[0].eval(),
second=10,
saveable=True,
name=save_name + str(epoch + 1),
fig_idx=2012)
except:
raise Exception(
"You should change visualize.W(), if you want to save the feature images for different dataset"
)
def fit(
sess, network, train_op, cost, X_train, y_train, x, y_, acc=None, batch_size=100, n_epoch=100, print_freq=5,
X_val=None, y_val=None, eval_train=True, tensorboard=False, tensorboard_epoch_freq=5,
tensorboard_weight_histograms=True, tensorboard_graph_vis=True
):
"""Training a given non time-series network by the given cost function, training data, batch_size, n_epoch etc.
- MNIST example click `here <https://github.com/tensorlayer/tensorlayer/blob/master/example/tutorial_mnist_simple.py>`_.
- In order to control the training details, the authors HIGHLY recommend ``tl.iterate`` see two MNIST examples `1 <https://github.com/tensorlayer/tensorlayer/blob/master/example/tutorial_mlp_dropout1.py>`_, `2 <https://github.com/tensorlayer/tensorlayer/blob/master/example/tutorial_mlp_dropout1.py>`_.
Parameters
----------
sess : Session
TensorFlow Session.
network : TensorLayer layer
the network to be trained.
train_op : TensorFlow optimizer
The optimizer for training e.g. tf.train.AdamOptimizer.
X_train : numpy.array
The input of training data
y_train : numpy.array
The target of training data
x : placeholder
For inputs.
y_ : placeholder
For targets.
acc : TensorFlow expression or None
Metric for accuracy or others. If None, would not print the information.
batch_size : int
The batch size for training and evaluating.
n_epoch : int
The number of training epochs.
print_freq : int
Print the training information every ``print_freq`` epochs.
X_val : numpy.array or None
The input of validation data. If None, would not perform validation.
y_val : numpy.array or None
The target of validation data. If None, would not perform validation.
eval_train : boolean
Whether to evaluate the model during training.
If X_val and y_val are not None, it reflects whether to evaluate the model on training data.
tensorboard : boolean
If True, summary data will be stored to the log/ directory for visualization with tensorboard.
See also detailed tensorboard_X settings for specific configurations of features. (default False)
Also runs `tl.layers.initialize_global_variables(sess)` internally in fit() to setup the summary nodes.
tensorboard_epoch_freq : int
How many epochs between storing tensorboard checkpoint for visualization to log/ directory (default 5).
tensorboard_weight_histograms : boolean
If True updates tensorboard data in the logs/ directory for visualization
of the weight histograms every tensorboard_epoch_freq epoch (default True).
tensorboard_graph_vis : boolean
If True stores the graph in the tensorboard summaries saved to log/ (default True).
Examples
--------
See `tutorial_mnist_simple.py <https://github.com/tensorlayer/tensorlayer/blob/master/example/tutorial_mnist_simple.py>`_
>>> tl.utils.fit(sess, network, train_op, cost, X_train, y_train, x, y_,
... acc=acc, batch_size=500, n_epoch=200, print_freq=5,
... X_val=X_val, y_val=y_val, eval_train=False)
>>> tl.utils.fit(sess, network, train_op, cost, X_train, y_train, x, y_,
... acc=acc, batch_size=500, n_epoch=200, print_freq=5,
... X_val=X_val, y_val=y_val, eval_train=False,
... tensorboard=True, tensorboard_weight_histograms=True, tensorboard_graph_vis=True)
Notes
--------
If tensorboard=True, the `global_variables_initializer` will be run inside the fit function
in order to initialize the automatically generated summary nodes used for tensorboard visualization,
thus `tf.global_variables_initializer().run()` before the `fit()` call will be undefined.
"""
if X_train.shape[0] < batch_size:
raise AssertionError("Number of training examples should be bigger than the batch size")
if (tensorboard):
logging.info("Setting up tensorboard ...")
#Set up tensorboard summaries and saver
tl.files.exists_or_mkdir('logs/')
#Only write summaries for more recent TensorFlow versions
if hasattr(tf, 'summary') and hasattr(tf.summary, 'FileWriter'):
if tensorboard_graph_vis:
train_writer = tf.summary.FileWriter('logs/train', sess.graph)
val_writer = tf.summary.FileWriter('logs/validation', sess.graph)
else:
train_writer = tf.summary.FileWriter('logs/train')
val_writer = tf.summary.FileWriter('logs/validation')
#Set up summary nodes
if (tensorboard_weight_histograms):
for param in network.all_params:
if hasattr(tf, 'summary') and hasattr(tf.summary, 'histogram'):
logging.info('Param name %s' % param.name)
tf.summary.histogram(param.name, param)
if hasattr(tf, 'summary') and hasattr(tf.summary, 'histogram'):
tf.summary.scalar('cost', cost)
merged = tf.summary.merge_all()
#Initalize all variables and summaries
tl.layers.initialize_global_variables(sess)
logging.info("Finished! use $tensorboard --logdir=logs/ to start server")
logging.info("Start training the network ...")
start_time_begin = time.time()
tensorboard_train_index, tensorboard_val_index = 0, 0
for epoch in range(n_epoch):
start_time = time.time()
loss_ep = 0
n_step = 0
for X_train_a, y_train_a in iterate.minibatches(X_train, y_train, batch_size, shuffle=True):
feed_dict = {x: X_train_a, y_: y_train_a}
feed_dict.update(network.all_drop) # enable noise layers
loss, _ = sess.run([cost, train_op], feed_dict=feed_dict)
loss_ep += loss
n_step += 1
loss_ep = loss_ep / n_step
if tensorboard and hasattr(tf, 'summary'):
if epoch + 1 == 1 or (epoch + 1) % tensorboard_epoch_freq == 0:
for X_train_a, y_train_a in iterate.minibatches(X_train, y_train, batch_size, shuffle=True):
dp_dict = dict_to_one(network.all_drop) # disable noise layers
feed_dict = {x: X_train_a, y_: y_train_a}
feed_dict.update(dp_dict)
result = sess.run(merged, feed_dict=feed_dict)
train_writer.add_summary(result, tensorboard_train_index)
tensorboard_train_index += 1
if (X_val is not None) and (y_val is not None):
for X_val_a, y_val_a in iterate.minibatches(X_val, y_val, batch_size, shuffle=True):
dp_dict = dict_to_one(network.all_drop) # disable noise layers
feed_dict = {x: X_val_a, y_: y_val_a}
feed_dict.update(dp_dict)
result = sess.run(merged, feed_dict=feed_dict)
val_writer.add_summary(result, tensorboard_val_index)
tensorboard_val_index += 1
if epoch + 1 == 1 or (epoch + 1) % print_freq == 0:
if (X_val is not None) and (y_val is not None):
logging.info("Epoch %d of %d took %fs" % (epoch + 1, n_epoch, time.time() - start_time))
if eval_train is True:
train_loss, train_acc, n_batch = 0, 0, 0
for X_train_a, y_train_a in iterate.minibatches(X_train, y_train, batch_size, shuffle=True):
dp_dict = dict_to_one(network.all_drop) # disable noise layers
feed_dict = {x: X_train_a, y_: y_train_a}
feed_dict.update(dp_dict)
if acc is not None:
err, ac = sess.run([cost, acc], feed_dict=feed_dict)
train_acc += ac
else:
err = sess.run(cost, feed_dict=feed_dict)
train_loss += err
n_batch += 1
logging.info(" train loss: %f" % (train_loss / n_batch))
if acc is not None:
logging.info(" train acc: %f" % (train_acc / n_batch))
val_loss, val_acc, n_batch = 0, 0, 0
for X_val_a, y_val_a in iterate.minibatches(X_val, y_val, batch_size, shuffle=True):
dp_dict = dict_to_one(network.all_drop) # disable noise layers
feed_dict = {x: X_val_a, y_: y_val_a}
feed_dict.update(dp_dict)
if acc is not None:
err, ac = sess.run([cost, acc], feed_dict=feed_dict)
val_acc += ac
else:
err = sess.run(cost, feed_dict=feed_dict)
val_loss += err
n_batch += 1
logging.info(" val loss: %f" % (val_loss / n_batch))
if acc is not None:
logging.info(" val acc: %f" % (val_acc / n_batch))
else:
logging.info(
"Epoch %d of %d took %fs, loss %f" % (epoch + 1, n_epoch, time.time() - start_time, loss_ep)
)
logging.info("Total training time: %fs" % (time.time() - start_time_begin))
def predict(sess, network, X, x, y_op, batch_size=None):
"""
Return the predict results of given non time-series network.
Parameters
----------
sess : Session
TensorFlow Session.
network : TensorLayer layer
The network.
X : numpy.array
The inputs.
x : placeholder
For inputs.
y_op : placeholder
The argmax expression of softmax outputs.
batch_size : int or None
The batch size for prediction, when dataset is large, we should use minibatche for prediction;
if dataset is small, we can set it to None.
Examples
--------
See `tutorial_mnist_simple.py <https://github.com/tensorlayer/tensorlayer/blob/master/example/tutorial_mnist_simple.py>`_
>>> y = network.outputs
>>> y_op = tf.argmax(tf.nn.softmax(y), 1)
>>> print(tl.utils.predict(sess, network, X_test, x, y_op))
"""
if batch_size is None:
dp_dict = dict_to_one(network.all_drop) # disable noise layers
feed_dict = {
x: X,
}
feed_dict.update(dp_dict)
return sess.run(y_op, feed_dict=feed_dict)
else:
result = None
for X_a, _ in iterate.minibatches(X, X, batch_size, shuffle=False):
dp_dict = dict_to_one(network.all_drop)
feed_dict = {
x: X_a,
}
feed_dict.update(dp_dict)
result_a = sess.run(y_op, feed_dict=feed_dict)
if result is None:
result = result_a
else:
result = np.concatenate((result, result_a))
if result is None:
if len(X) % batch_size != 0:
dp_dict = dict_to_one(network.all_drop)
feed_dict = {
x: X[-(len(X) % batch_size):, :],
}
feed_dict.update(dp_dict)
result_a = sess.run(y_op, feed_dict=feed_dict)
result = result_a
else:
if len(X) != len(result) and len(X) % batch_size != 0:
dp_dict = dict_to_one(network.all_drop)
feed_dict = {
x: X[-(len(X) % batch_size):, :],
}
feed_dict.update(dp_dict)
result_a = sess.run(y_op, feed_dict=feed_dict)
result = np.concatenate((result, result_a))
return result
