def pretrain(self, sess, x, X_train, X_val, denoise_name=None, n_epoch=100, batch_size=128, print_freq=10, save=True, save_name='w1pre_'): ''' pre-train the parameters of previous DenseLayer ''' print(" tensorlayer: %s start pretrain" % self.name) print(" batch_size: %d" % batch_size) if denoise_name: print(" denoising layer keep: %f" % self.all_drop[set_keep[denoise_name]]) dp_denoise = self.all_drop[set_keep[denoise_name]] else: print(" no denoising layer") # You may want to check different cost values # dp_dict = utils.dict_to_one( self.all_drop ) # feed_dict = {x: X_val} # feed_dict.update(dp_dict) # print(sess.run([self.mse, self.L1_a, self.L2_w], feed_dict=feed_dict)) # exit() for epoch in range(n_epoch): start_time = time.time() for X_train_a, _ in iterate.minibatches(X_train, X_train, batch_size, shuffle=True): dp_dict = utils.dict_to_one( self.all_drop ) if denoise_name: dp_dict[set_keep[denoise_name]] = dp_denoise feed_dict = {x: X_train_a} feed_dict.update(dp_dict) sess.run(self.train_op, feed_dict=feed_dict) if epoch + 1 == 1 or (epoch + 1) % print_freq == 0: print("Epoch %d of %d took %fs" % (epoch + 1, n_epoch, time.time() - start_time)) train_loss, n_batch = 0, 0 for X_train_a, _ in iterate.minibatches(X_train, X_train, batch_size, shuffle=True): dp_dict = utils.dict_to_one( self.all_drop ) feed_dict = {x: X_train_a} feed_dict.update(dp_dict) err = sess.run(self.cost, feed_dict=feed_dict) train_loss += err n_batch += 1 print(" train loss: %f" % (train_loss/ n_batch)) val_loss, n_batch = 0, 0 for X_val_a, _ in iterate.minibatches(X_val, X_val, batch_size, shuffle=True): dp_dict = utils.dict_to_one( self.all_drop ) feed_dict = {x: X_val_a} feed_dict.update(dp_dict) err = sess.run(self.cost, feed_dict=feed_dict) val_loss += err n_batch += 1 print(" val loss: %f" % (val_loss/ n_batch)) if save: try: visualize.W(self.train_params[0].eval(), second=10, saveable=True, name=save_name+str(epoch+1), fig_idx=2012) except: raise Exception("You should change visualize.W(), if you want to save the feature images for different dataset")
def pretrain(self, sess, x, X_train, X_val, denoise_name=None, n_epoch=100, batch_size=128, print_freq=10, save=True, save_name='w1pre_'): # ==================================================== # # You need to modify the cost function in __init__() so as to # get your own pre-train method. # # ==================================================== print(" tensorlayer: %s start pretrain" % self.name) print(" batch_size: %d" % batch_size) if denoise_name: print(" denoising layer keep: %f" % self.all_drop[set_keep[denoise_name]]) dp_denoise = self.all_drop[set_keep[denoise_name]] else: print(" no denoising layer") for epoch in range(n_epoch): start_time = time.time() for X_train_a, _ in iterate.minibatches(X_train, X_train, batch_size, shuffle=True): dp_dict = utils.dict_to_one( self.all_drop ) if denoise_name: dp_dict[set_keep[denoise_name]] = dp_denoise feed_dict = {x: X_train_a} feed_dict.update(dp_dict) sess.run(self.train_op, feed_dict=feed_dict) if epoch + 1 == 1 or (epoch + 1) % print_freq == 0: print("Epoch %d of %d took %fs" % (epoch + 1, n_epoch, time.time() - start_time)) train_loss, n_batch = 0, 0 for X_train_a, _ in iterate.minibatches(X_train, X_train, batch_size, shuffle=True): dp_dict = utils.dict_to_one( self.all_drop ) feed_dict = {x: X_train_a} feed_dict.update(dp_dict) err = sess.run(self.cost, feed_dict=feed_dict) train_loss += err n_batch += 1 print(" train loss: %f" % (train_loss/ n_batch)) val_loss, n_batch = 0, 0 for X_val_a, _ in iterate.minibatches(X_val, X_val, batch_size, shuffle=True): dp_dict = utils.dict_to_one( self.all_drop ) feed_dict = {x: X_val_a} feed_dict.update(dp_dict) err = sess.run(self.cost, feed_dict=feed_dict) val_loss += err n_batch += 1 print(" val loss: %f" % (val_loss/ n_batch)) if save: try: visualize.W(self.train_params[0].eval(), second=10, saveable=True, shape=[28,28], name=save_name+str(epoch+1), fig_idx=2012) files.save_npz([self.all_params[0]] , name=save_name+str(epoch+1)+'.npz') except: raise Exception("You should change visualize.W(), if you want to save the feature images for different dataset")
def test(sess, network, acc, X_test, y_test, x, y_, batch_size, cost=None): """ Test a given non time-series network by the given test data and metric. Parameters ---------- sess : Session TensorFlow session. network : TensorLayer layer The network. acc : TensorFlow expression or None Metric for accuracy or others. - If None, would not print the information. X_test : numpy.array The input of testing data. y_test : numpy array The target of testing data x : placeholder For inputs. y_ : placeholder For targets. batch_size : int or None The batch size for testing, when dataset is large, we should use minibatche for testing; if dataset is small, we can set it to None. cost : TensorFlow expression or None Metric for cost or others. If None, would not print the information. Examples -------- See `tutorial_mnist_simple.py <https://github.com/tensorlayer/tensorlayer/blob/master/example/tutorial_mnist_simple.py>`_ >>> tl.utils.test(sess, network, acc, X_test, y_test, x, y_, batch_size=None, cost=cost) """ logging.info('Start testing the network ...') if batch_size is None: dp_dict = dict_to_one(network.all_drop) feed_dict = {x: X_test, y_: y_test} feed_dict.update(dp_dict) if cost is not None: logging.info(" test loss: %f" % sess.run(cost, feed_dict=feed_dict)) logging.info(" test acc: %f" % sess.run(acc, feed_dict=feed_dict)) # logging.info(" test acc: %f" % np.mean(y_test == sess.run(y_op, # feed_dict=feed_dict))) else: test_loss, test_acc, n_batch = 0, 0, 0 for X_test_a, y_test_a in iterate.minibatches(X_test, y_test, batch_size, shuffle=True): dp_dict = dict_to_one(network.all_drop) # disable noise layers feed_dict = {x: X_test_a, y_: y_test_a} feed_dict.update(dp_dict) if cost is not None: err, ac = sess.run([cost, acc], feed_dict=feed_dict) test_loss += err else: ac = sess.run(acc, feed_dict=feed_dict) test_acc += ac n_batch += 1 if cost is not None: logging.info(" test loss: %f" % (test_loss / n_batch)) logging.info(" test acc: %f" % (test_acc / n_batch))
def save_images(self, nets, epoch, curr_tr, images_i, images_j): """ Saves input and output images. """ nets = utils.set_mode(nets, "eval") exists_or_mkdir(self._images_dir) if curr_tr > 0: donorm = False else: donorm = True names = [ 'inputA_', 'inputB_', 'fakeA_', 'fakeB_', 'cycA_', 'cycB_', 'mask_a', 'mask_b' ] with open( os.path.join(self._output_dir, 'epoch_' + str(epoch) + '.html'), 'w') as v_html: input_iter = minibatches(images_i, images_j, batch_size=1, shuffle=True) for i in range(0, self._num_imgs_to_save): #pdb.set_trace() print("Saving image {}/{}...".format(i, self._num_imgs_to_save)) self.image_a, self.image_b = next(input_iter) tmp_imgA = self.get_fake_image_pool(self.num_fake_inputs, self.fake_images_A) self.fake_pool_A_mask = tmp_imgA["mask"] self.fake_pool_A = tmp_imgA["im"] tmp_imgB = self.get_fake_image_pool(self.num_fake_inputs, self.fake_images_B) self.fake_pool_B_mask = tmp_imgB["mask"] self.fake_pool_B = tmp_imgB["im"] self.transition_rate = np.array([curr_tr], dtype=np.float32) self.donorm = np.array([donorm], dtype=np.float32) self.output_converter( model.get_outputs(self.input_converter(), nets)) figures_to_save = [ self.image_a, self.image_b, self.fake_images_b, self.fake_images_a, self.cycle_images_a, self.cycle_images_b, self.masks[0], self.masks[1] ] self.figure_writer(figures_to_save, names, v_html, epoch, i, html_mode=0)
def pretrain(self, sess, x, X_train, X_val, denoise_name=None, n_epoch=100, batch_size=128, print_freq=10, save=True, save_name='w1pre_'): # ==================================================== # # You need to modify the cost function in __init__() so as to # get your own pre-train method. # # ==================================================== logging.info(" [*] %s start pretrain" % self.name) logging.info(" batch_size: %d" % batch_size) if denoise_name: logging.info(" denoising layer keep: %f" % self.all_drop[LayersConfig.set_keep[denoise_name]]) dp_denoise = self.all_drop[LayersConfig.set_keep[denoise_name]] else: logging.info(" no denoising layer") for epoch in range(n_epoch): start_time = time.time() for X_train_a, _ in iterate.minibatches(X_train, X_train, batch_size, shuffle=True): dp_dict = utils.dict_to_one(self.all_drop) if denoise_name: dp_dict[LayersConfig.set_keep[denoise_name]] = dp_denoise feed_dict = {x: X_train_a} feed_dict.update(dp_dict) sess.run(self.train_op, feed_dict=feed_dict) if epoch + 1 == 1 or (epoch + 1) % print_freq == 0: logging.info("Epoch %d of %d took %fs" % (epoch + 1, n_epoch, time.time() - start_time)) train_loss, n_batch = 0, 0 for X_train_a, _ in iterate.minibatches(X_train, X_train, batch_size, shuffle=True): dp_dict = utils.dict_to_one(self.all_drop) feed_dict = {x: X_train_a} feed_dict.update(dp_dict) err = sess.run(self.cost, feed_dict=feed_dict) train_loss += err n_batch += 1 logging.info(" train loss: %f" % (train_loss / n_batch)) val_loss, n_batch = 0, 0 for X_val_a, _ in iterate.minibatches(X_val, X_val, batch_size, shuffle=True): dp_dict = utils.dict_to_one(self.all_drop) feed_dict = {x: X_val_a} feed_dict.update(dp_dict) err = sess.run(self.cost, feed_dict=feed_dict) val_loss += err n_batch += 1 logging.info(" val loss: %f" % (val_loss / n_batch)) if save: try: visualize.draw_weights(self.train_params[0].eval(), second=10, saveable=True, shape=[28, 28], name=save_name + str(epoch + 1), fig_idx=2012) files.save_npz([self.all_params[0]], name=save_name + str(epoch + 1) + '.npz') except Exception: raise Exception( "You should change the visualize.W() in ReconLayer.pretrain(), if you want to save the feature images for different dataset" )
def train(self): """ Training Function. We use batch size = 1 for training """ # Build the network and compute losses nets = self.model_setup() summary_writer = tf.summary.create_file_writer( os.path.join(self._output_dir, "log")) summary_writer.set_as_default() max_images = cyclegan_datasets.DATASET_TO_SIZES[self._dataset_name] half_training_ep = int(self._max_step / 2) # Restore the model to run the model from last checkpoint print("Loading the latest checkpoint...") if self._to_restore: checkpoint_name = os.path.join(self._checkpoint_dir, self._checkpoint_name) utils.load(checkpoint_name, nets=nets) self.global_step = int(checkpoint_name[-2:]) else: self.global_step = 0 exists_or_mkdir(self._output_dir) self.upd_fake_image_pool(self.num_fake_inputs, self.fake_pool_A, self.fake_pool_A_mask, self.fake_images_A) self.upd_fake_image_pool(self.num_fake_inputs, self.fake_pool_B, self.fake_pool_B_mask, self.fake_images_B) self.num_fake_inputs += 1 # Training Loop for epoch in range(self.global_step, self._max_step): print("In the epoch ", epoch) print("Saving the latest checkpoint...") utils.save(nets, os.path.join(self._output_dir, "AGGAN_%02d" % epoch)) # Setting lr curr_lr = self._base_lr if epoch >= half_training_ep: curr_lr -= self._base_lr * ( epoch - half_training_ep) / half_training_ep self.g_A_trainer.learning_rate = curr_lr self.g_B_trainer.learning_rate = curr_lr self.g_A_trainer_bis.learning_rate = curr_lr self.g_B_trainer_bis.learning_rate = curr_lr self.d_A_trainer.learning_rate = curr_lr self.d_B_trainer.learning_rate = curr_lr if epoch < self._switch: curr_tr = 0 donorm = True to_train_A = self.g_A_trainer to_train_B = self.g_B_trainer to_train_A_vars = self.g_A_vars + self.g_Ae_vars to_train_B_vars = self.g_B_vars + self.g_Be_vars else: curr_tr = self._threshold_fg donorm = False to_train_A = self.g_A_trainer_bis to_train_B = self.g_B_trainer_bis to_train_A_vars = self.g_A_vars to_train_B_vars = self.g_B_vars print("Loading data...") tot_inputs = data_loader.load_data(self._dataset_name, self._size_before_crop, False, self._do_flipping) self.inputs_img_i = tot_inputs['images_i'] self.inputs_img_j = tot_inputs['images_j'] assert (len(self.inputs_img_i) == len(self.inputs_img_j) and max_images == len(self.inputs_img_i)) self.save_images(nets, epoch, curr_tr, self.inputs_img_i, self.inputs_img_j) nets = utils.set_mode(nets, "train") input_iter = minibatches(self.inputs_img_i, self.inputs_img_j, batch_size=1, shuffle=True) for i in range(max_images): print("Processing batch {}/{} in {}th epoch".format( i, max_images, epoch)) self.image_a, self.image_b = next(input_iter) tmp_imgA = self.get_fake_image_pool(self.num_fake_inputs, self.fake_images_A) self.fake_pool_A_mask = tmp_imgA["mask"] self.fake_pool_A = tmp_imgA["im"] tmp_imgB = self.get_fake_image_pool(self.num_fake_inputs, self.fake_images_B) self.fake_pool_B_mask = tmp_imgB["mask"] self.fake_pool_B = tmp_imgB["im"] self.transition_rate = np.array([curr_tr], dtype=np.float32) self.donorm = np.array([donorm], dtype=np.float32) with tf.GradientTape(persistent=True) as tape: self.output_converter( model.get_outputs(self.input_converter(), nets)) self.upd_fake_image_pool(self.num_fake_inputs, self.fake_images_b, self.masks[0], self.fake_images_B) self.upd_fake_image_pool(self.num_fake_inputs, self.fake_images_a, self.masks[1], self.fake_images_A) self.compute_losses() #pdb.set_trace() grad = tape.gradient(self.d_B_loss, self.d_B_vars) self.d_B_trainer.apply_gradients(zip(grad, self.d_B_vars)) grad = tape.gradient(self.d_A_loss, self.d_A_vars) self.d_A_trainer.apply_gradients(zip(grad, self.d_A_vars)) grad = tape.gradient(self.g_A_loss, to_train_A_vars) to_train_A.apply_gradients(zip(grad, to_train_A_vars)) grad = tape.gradient(self.g_B_loss, to_train_B_vars) to_train_B.apply_gradients(zip(grad, to_train_B_vars)) tot_loss = self.g_A_loss + self.g_B_loss + self.d_A_loss + self.d_B_loss print("[training_info] g_A_loss = {}, g_B_loss = {}, d_A_loss = {}, d_B_loss = {}, \ tot_loss = {}, lr={}, curr_tr={}" .format(self.g_A_loss, self.g_B_loss, self.d_A_loss, \ self.d_B_loss, tot_loss, curr_lr, curr_tr)) tf.summary.scalar('g_A_loss', self.g_A_loss, step=self.global_step * max_images + i) tf.summary.scalar('g_B_loss', self.g_B_loss, step=self.global_step * max_images + i) tf.summary.scalar('d_A_loss', self.d_A_loss, step=self.global_step * max_images + i) tf.summary.scalar('d_B_loss', self.d_B_loss, step=self.global_step * max_images + i) tf.summary.scalar('learning_rate', to_train_A.learning_rate, step=self.global_step * max_images + i) tf.summary.scalar('total_loss', tot_loss, step=self.global_step * max_images + i) self.num_fake_inputs += 1 self.global_step = epoch + 1 summary_writer.flush()
def save_images_bis(self, nets, epoch, images_i, images_j): """ Saves input and output images. """ names = [ 'input_A_', 'mask_A_', 'masked_inputA_', 'fakeB_', 'input_B_', 'mask_B_', 'masked_inputB_', 'fakeA_' ] space = '                        ' \ '                        ' \ '         ' nets = utils.set_mode(nets, "eval") #pdb.set_trace() exists_or_mkdir(self._images_dir) with open( os.path.join(self._output_dir, 'results_' + str(epoch) + '.html'), 'w') as v_html: v_html.write("<b>Inputs" + space + "Masks" + space + "Masked_images" + space + "Generated_images</b>") v_html.write("<br>") input_iter = minibatches(images_i, images_j, batch_size=1, shuffle=True) for i in range(0, self._num_imgs_to_save): print("Saving image {}/{}...".format(i, self._num_imgs_to_save)) #pdb.set_trace() self.image_a, self.image_b = next(input_iter) tmp_imgA = self.get_fake_image_pool(self.num_fake_inputs, self.fake_images_A) self.fake_pool_A_mask = tmp_imgA["mask"] self.fake_pool_A = tmp_imgA["im"] tmp_imgB = self.get_fake_image_pool(self.num_fake_inputs, self.fake_images_B) self.fake_pool_B_mask = tmp_imgB["mask"] self.fake_pool_B = tmp_imgB["im"] self.transition_rate = np.array([0.1], dtype=np.float32) self.donorm = np.array([True], dtype=np.float32) self.output_converter( model.get_outputs(self.input_converter(), nets)) figures_to_save = [ self.image_a, self.masks[0], self.masked_ims[0], self.fake_images_b, self.image_b, self.masks[1], self.masked_ims[1], self.fake_images_a ] self.figure_writer(figures_to_save, names, v_html, epoch, i, html_mode=1)
sess.run(init_op) # Run training loop for layer in x_model.layers: layer.trainable = False with sess.as_default(): #saver = tf.train.Saver() if goon: #saver.restore(sess,'./model.ckpt') x_model.load_weights('x_model.h5') print('OK load!!!!!!!!!!') else: print('Creat new!!!!!!!!!!') for i in range(100): for X_train_a, y_train_a in minibatches(X_train, y_train, batch_size, shuffle=True): _, _ = sess.run([cost, train_op], feed_dict={ img: X_train_a, y_: y_train_a, K.learning_phase(): 1 }) train_loss, train_acc, n_batch = 0, 0, 0 for X_train_a, y_train_a in minibatches(X_train, y_train, batch_size, shuffle=False): err, ac = sess.run([cost, acc], feed_dict={
def pretrain(self, sess, x, X_train, X_val, denoise_name=None, n_epoch=100, batch_size=128, print_freq=10, save=True, save_name='w1pre_'): ''' pre-train the parameters of previous DenseLayer ''' print(" tensorlayer: %s start pretrain" % self.name) print(" batch_size: %d" % batch_size) if denoise_name: print(" denoising layer keep: %f" % self.all_drop[set_keep[denoise_name]]) dp_denoise = self.all_drop[set_keep[denoise_name]] else: print(" no denoising layer") # You may want to check different cost values # dp_dict = utils.dict_to_one( self.all_drop ) # feed_dict = {x: X_val} # feed_dict.update(dp_dict) # print(sess.run([self.mse, self.L1_a, self.L2_w], feed_dict=feed_dict)) # exit() for epoch in range(n_epoch): start_time = time.time() for X_train_a, _ in iterate.minibatches(X_train, X_train, batch_size, shuffle=True): dp_dict = utils.dict_to_one(self.all_drop) if denoise_name: dp_dict[set_keep[denoise_name]] = dp_denoise feed_dict = {x: X_train_a} feed_dict.update(dp_dict) sess.run(self.train_op, feed_dict=feed_dict) if epoch + 1 == 1 or (epoch + 1) % print_freq == 0: print("Epoch %d of %d took %fs" % (epoch + 1, n_epoch, time.time() - start_time)) train_loss, n_batch = 0, 0 for X_train_a, _ in iterate.minibatches(X_train, X_train, batch_size, shuffle=True): dp_dict = utils.dict_to_one(self.all_drop) feed_dict = {x: X_train_a} feed_dict.update(dp_dict) err = sess.run(self.cost, feed_dict=feed_dict) train_loss += err n_batch += 1 print(" train loss: %f" % (train_loss / n_batch)) val_loss, n_batch = 0, 0 for X_val_a, _ in iterate.minibatches(X_val, X_val, batch_size, shuffle=True): dp_dict = utils.dict_to_one(self.all_drop) feed_dict = {x: X_val_a} feed_dict.update(dp_dict) err = sess.run(self.cost, feed_dict=feed_dict) val_loss += err n_batch += 1 print(" val loss: %f" % (val_loss / n_batch)) if save: try: visualize.W(self.train_params[0].eval(), second=10, saveable=True, name=save_name + str(epoch + 1), fig_idx=2012) except: raise Exception( "You should change visualize.W(), if you want to save the feature images for different dataset" )
def fit( sess, network, train_op, cost, X_train, y_train, x, y_, acc=None, batch_size=100, n_epoch=100, print_freq=5, X_val=None, y_val=None, eval_train=True, tensorboard=False, tensorboard_epoch_freq=5, tensorboard_weight_histograms=True, tensorboard_graph_vis=True ): """Training a given non time-series network by the given cost function, training data, batch_size, n_epoch etc. - MNIST example click `here <https://github.com/tensorlayer/tensorlayer/blob/master/example/tutorial_mnist_simple.py>`_. - In order to control the training details, the authors HIGHLY recommend ``tl.iterate`` see two MNIST examples `1 <https://github.com/tensorlayer/tensorlayer/blob/master/example/tutorial_mlp_dropout1.py>`_, `2 <https://github.com/tensorlayer/tensorlayer/blob/master/example/tutorial_mlp_dropout1.py>`_. Parameters ---------- sess : Session TensorFlow Session. network : TensorLayer layer the network to be trained. train_op : TensorFlow optimizer The optimizer for training e.g. tf.train.AdamOptimizer. X_train : numpy.array The input of training data y_train : numpy.array The target of training data x : placeholder For inputs. y_ : placeholder For targets. acc : TensorFlow expression or None Metric for accuracy or others. If None, would not print the information. batch_size : int The batch size for training and evaluating. n_epoch : int The number of training epochs. print_freq : int Print the training information every ``print_freq`` epochs. X_val : numpy.array or None The input of validation data. If None, would not perform validation. y_val : numpy.array or None The target of validation data. If None, would not perform validation. eval_train : boolean Whether to evaluate the model during training. If X_val and y_val are not None, it reflects whether to evaluate the model on training data. tensorboard : boolean If True, summary data will be stored to the log/ directory for visualization with tensorboard. See also detailed tensorboard_X settings for specific configurations of features. (default False) Also runs `tl.layers.initialize_global_variables(sess)` internally in fit() to setup the summary nodes. tensorboard_epoch_freq : int How many epochs between storing tensorboard checkpoint for visualization to log/ directory (default 5). tensorboard_weight_histograms : boolean If True updates tensorboard data in the logs/ directory for visualization of the weight histograms every tensorboard_epoch_freq epoch (default True). tensorboard_graph_vis : boolean If True stores the graph in the tensorboard summaries saved to log/ (default True). Examples -------- See `tutorial_mnist_simple.py <https://github.com/tensorlayer/tensorlayer/blob/master/example/tutorial_mnist_simple.py>`_ >>> tl.utils.fit(sess, network, train_op, cost, X_train, y_train, x, y_, ... acc=acc, batch_size=500, n_epoch=200, print_freq=5, ... X_val=X_val, y_val=y_val, eval_train=False) >>> tl.utils.fit(sess, network, train_op, cost, X_train, y_train, x, y_, ... acc=acc, batch_size=500, n_epoch=200, print_freq=5, ... X_val=X_val, y_val=y_val, eval_train=False, ... tensorboard=True, tensorboard_weight_histograms=True, tensorboard_graph_vis=True) Notes -------- If tensorboard=True, the `global_variables_initializer` will be run inside the fit function in order to initialize the automatically generated summary nodes used for tensorboard visualization, thus `tf.global_variables_initializer().run()` before the `fit()` call will be undefined. """ if X_train.shape[0] < batch_size: raise AssertionError("Number of training examples should be bigger than the batch size") if (tensorboard): logging.info("Setting up tensorboard ...") #Set up tensorboard summaries and saver tl.files.exists_or_mkdir('logs/') #Only write summaries for more recent TensorFlow versions if hasattr(tf, 'summary') and hasattr(tf.summary, 'FileWriter'): if tensorboard_graph_vis: train_writer = tf.summary.FileWriter('logs/train', sess.graph) val_writer = tf.summary.FileWriter('logs/validation', sess.graph) else: train_writer = tf.summary.FileWriter('logs/train') val_writer = tf.summary.FileWriter('logs/validation') #Set up summary nodes if (tensorboard_weight_histograms): for param in network.all_params: if hasattr(tf, 'summary') and hasattr(tf.summary, 'histogram'): logging.info('Param name %s' % param.name) tf.summary.histogram(param.name, param) if hasattr(tf, 'summary') and hasattr(tf.summary, 'histogram'): tf.summary.scalar('cost', cost) merged = tf.summary.merge_all() #Initalize all variables and summaries tl.layers.initialize_global_variables(sess) logging.info("Finished! use $tensorboard --logdir=logs/ to start server") logging.info("Start training the network ...") start_time_begin = time.time() tensorboard_train_index, tensorboard_val_index = 0, 0 for epoch in range(n_epoch): start_time = time.time() loss_ep = 0 n_step = 0 for X_train_a, y_train_a in iterate.minibatches(X_train, y_train, batch_size, shuffle=True): feed_dict = {x: X_train_a, y_: y_train_a} feed_dict.update(network.all_drop) # enable noise layers loss, _ = sess.run([cost, train_op], feed_dict=feed_dict) loss_ep += loss n_step += 1 loss_ep = loss_ep / n_step if tensorboard and hasattr(tf, 'summary'): if epoch + 1 == 1 or (epoch + 1) % tensorboard_epoch_freq == 0: for X_train_a, y_train_a in iterate.minibatches(X_train, y_train, batch_size, shuffle=True): dp_dict = dict_to_one(network.all_drop) # disable noise layers feed_dict = {x: X_train_a, y_: y_train_a} feed_dict.update(dp_dict) result = sess.run(merged, feed_dict=feed_dict) train_writer.add_summary(result, tensorboard_train_index) tensorboard_train_index += 1 if (X_val is not None) and (y_val is not None): for X_val_a, y_val_a in iterate.minibatches(X_val, y_val, batch_size, shuffle=True): dp_dict = dict_to_one(network.all_drop) # disable noise layers feed_dict = {x: X_val_a, y_: y_val_a} feed_dict.update(dp_dict) result = sess.run(merged, feed_dict=feed_dict) val_writer.add_summary(result, tensorboard_val_index) tensorboard_val_index += 1 if epoch + 1 == 1 or (epoch + 1) % print_freq == 0: if (X_val is not None) and (y_val is not None): logging.info("Epoch %d of %d took %fs" % (epoch + 1, n_epoch, time.time() - start_time)) if eval_train is True: train_loss, train_acc, n_batch = 0, 0, 0 for X_train_a, y_train_a in iterate.minibatches(X_train, y_train, batch_size, shuffle=True): dp_dict = dict_to_one(network.all_drop) # disable noise layers feed_dict = {x: X_train_a, y_: y_train_a} feed_dict.update(dp_dict) if acc is not None: err, ac = sess.run([cost, acc], feed_dict=feed_dict) train_acc += ac else: err = sess.run(cost, feed_dict=feed_dict) train_loss += err n_batch += 1 logging.info(" train loss: %f" % (train_loss / n_batch)) if acc is not None: logging.info(" train acc: %f" % (train_acc / n_batch)) val_loss, val_acc, n_batch = 0, 0, 0 for X_val_a, y_val_a in iterate.minibatches(X_val, y_val, batch_size, shuffle=True): dp_dict = dict_to_one(network.all_drop) # disable noise layers feed_dict = {x: X_val_a, y_: y_val_a} feed_dict.update(dp_dict) if acc is not None: err, ac = sess.run([cost, acc], feed_dict=feed_dict) val_acc += ac else: err = sess.run(cost, feed_dict=feed_dict) val_loss += err n_batch += 1 logging.info(" val loss: %f" % (val_loss / n_batch)) if acc is not None: logging.info(" val acc: %f" % (val_acc / n_batch)) else: logging.info( "Epoch %d of %d took %fs, loss %f" % (epoch + 1, n_epoch, time.time() - start_time, loss_ep) ) logging.info("Total training time: %fs" % (time.time() - start_time_begin))
def predict(sess, network, X, x, y_op, batch_size=None): """ Return the predict results of given non time-series network. Parameters ---------- sess : Session TensorFlow Session. network : TensorLayer layer The network. X : numpy.array The inputs. x : placeholder For inputs. y_op : placeholder The argmax expression of softmax outputs. batch_size : int or None The batch size for prediction, when dataset is large, we should use minibatche for prediction; if dataset is small, we can set it to None. Examples -------- See `tutorial_mnist_simple.py <https://github.com/tensorlayer/tensorlayer/blob/master/example/tutorial_mnist_simple.py>`_ >>> y = network.outputs >>> y_op = tf.argmax(tf.nn.softmax(y), 1) >>> print(tl.utils.predict(sess, network, X_test, x, y_op)) """ if batch_size is None: dp_dict = dict_to_one(network.all_drop) # disable noise layers feed_dict = { x: X, } feed_dict.update(dp_dict) return sess.run(y_op, feed_dict=feed_dict) else: result = None for X_a, _ in iterate.minibatches(X, X, batch_size, shuffle=False): dp_dict = dict_to_one(network.all_drop) feed_dict = { x: X_a, } feed_dict.update(dp_dict) result_a = sess.run(y_op, feed_dict=feed_dict) if result is None: result = result_a else: result = np.concatenate((result, result_a)) if result is None: if len(X) % batch_size != 0: dp_dict = dict_to_one(network.all_drop) feed_dict = { x: X[-(len(X) % batch_size):, :], } feed_dict.update(dp_dict) result_a = sess.run(y_op, feed_dict=feed_dict) result = result_a else: if len(X) != len(result) and len(X) % batch_size != 0: dp_dict = dict_to_one(network.all_drop) feed_dict = { x: X[-(len(X) % batch_size):, :], } feed_dict.update(dp_dict) result_a = sess.run(y_op, feed_dict=feed_dict) result = np.concatenate((result, result_a)) return result