def train_pro_gan(): """ Train the generative adverserial network. """ # Graph definition: inputs, model, loss, optimizer, initializer print('Graph building...') # Inputs image_inputs = tf.compat.v1.placeholder(tf.compat.v1.float32, shape=[None]+config.image_shape, name='image_inputs') noise_inputs = tf.compat.v1.placeholder(tf.compat.v1.float32, shape=[None,config.latent_size], name='noise_inputs') res_training = tf.compat.v1.placeholder(tf.compat.v1.float32, shape=[], name='res_training') minibatch_size = tf.compat.v1.placeholder(tf.compat.v1.int64, shape=[], name='minibatch_size') # Network gen_loss, disc_loss, output_images = getattr(network, config.network)( image_inputs, noise_inputs, latent_size=config.latent_size, minibatch_size=minibatch_size, res_building=int(np.log2(config.image_size)), res_training=res_training) var_list = tf.compat.v1.global_variables() # Store the list of variables restore_list = [] for v in var_list: if not '8' in v.name and not 'Output_6' in v.name: restore_list += [v] print(v) # Optimizer lr = tf.compat.v1.placeholder(tf.compat.v1.float32, name='lr') optimizer_disc = tf.compat.v1.train.AdamOptimizer(learning_rate=lr, **config.optimzer_kwargs) optimizer_gen = tf.compat.v1.train.AdamOptimizer(learning_rate=lr, **config.optimzer_kwargs) disc_vars = tf.compat.v1.get_collection(tf.compat.v1.GraphKeys.TRAINABLE_VARIABLES, scope='discriminator') gen_vars = tf.compat.v1.get_collection(tf.compat.v1.GraphKeys.TRAINABLE_VARIABLES, scope='generator') training_op_disc = optimizer_disc.minimize(disc_loss, var_list=disc_vars, name='training_op_disc') training_op_gen = optimizer_gen.minimize(gen_loss, var_list=gen_vars, name='training_op_gen') # Initializer init = tf.compat.v1.global_variables_initializer() print('Done: graph built.') # Init dataset inputs = getattr(dataset, config.data_initilize)(**config.data_initilize_kwargs) # Define the minibatch selector select_minibatch = partial(getattr(dataset, config.data_selector), inputs) # Saver saver = train.Saver(var_list,config.logs_path, {'gen_loss':gen_loss, 'disc_loss':disc_loss}) # Save minibatch test minibatch = select_minibatch(crt_img=0, res=config.image_size, minibatch_size=config.nbof_test_sample)/2 +0.5 saver.save_images(minibatch, 'test') # Time measurements graph_time = time() # Training --> use the configuration file print('Training...') with tf.compat.v1.Session() as sess: # Initialize init.run() # Restore former parameters if config.restore: print('Restoring weight stored in {}'.format(config.restore)) tf.compat.v1.train.Saver(var_list=restore_list).restore(sess, config.restore) # Training parameters noise_test = 2*np.random.random((config.nbof_test_sample, config.latent_size))-1 # Training ... cur_img = config.start_img while cur_img < config.end_img: # Change input dataset cur_res = training_schedule(cur_img) cur_minibatch_size = config.minibatch_size[int(np.ceil(cur_res))] minibatch = select_minibatch(crt_img=cur_img, res=2**int(np.ceil(cur_res)), minibatch_size=cur_minibatch_size) noises = 2*np.random.random((cur_minibatch_size, config.latent_size))-1 feed_dict = {} feed_dict['image_inputs:0'] = minibatch feed_dict['noise_inputs:0'] = noises feed_dict['lr:0'] = config.learning_rate feed_dict['res_training:0'] = cur_res feed_dict['minibatch_size:0'] = cur_minibatch_size sess.run([training_op_disc], feed_dict=feed_dict) sess.run([training_op_gen], feed_dict=feed_dict) # Display time information if cur_img % config.img_per_images == 0: graph_time = time() - graph_time minutes = int(graph_time // 60) secondes = graph_time - minutes * 60 print('{} kimgs: {:4d} minutes {:2f} secondes, {:2f} resolution'.format(cur_img//1000, minutes, secondes, cur_res)) graph_time = time() # Save logs if cur_img % config.img_per_summary == 0: saver.save_summary(sess, feed_dict, cur_img) # Save images if cur_img % config.img_per_images == 0: feed_dict = {noise_inputs:noise_test, res_training:cur_res} outputs = output_images.eval(feed_dict=feed_dict) outputs = outputs / 2 + 0.5 # Re scale output images outputs = np.clip(outputs, 0, 1) saver.save_images(outputs, cur_img//1000) # Save model if cur_img % config.img_per_save == 0: saver.save_model(sess, cur_img//1000) # Update current image cur_img += cur_minibatch_size # Final Saving saver.save_model(sess, 'final') saver.close_summary() print('Done: training')
def train_recent_from_npy(): """ Train the classification or the recognition network mode is in ['classification', 'recognition'] """ # Prepare the dataset training_filenames, training_labels_bytes = np.load(config.data_path) training_labels = np.array([ int(tlb.decode('ascii')) for tlb in training_labels_bytes ]) # Convert labels type: from bytes to int validation_filenames, validation_labels_bytes = np.load( config.data_test_path) validation_labels = np.array([ int(vlb.decode('ascii')) for vlb in validation_labels_bytes ]) # Convert labels type: from bytes to int # Graph definition: inputs, model, loss, optimizer, initializer print('Graph building...') # Small graph for decoding jpegs with tf.compat.v1.variable_scope('DecodeJPEG'): image_jpeg = tf.compat.v1.placeholder(tf.string, shape=None, name='images_raw') image_decoded = tf.io.decode_jpeg(image_jpeg) # Inputs images = tf.compat.v1.placeholder( tf.float32, shape=[None, config.image_size, config.image_size, 3], name='image_inputs') label_inputs = tf.compat.v1.placeholder(tf.int64, shape=[ None, ], name='label_inputs') training = tf.placeholder(tf.bool, shape=[], name='training') lr = tf.compat.v1.placeholder(tf.float32, shape=[], name='learning_rate') # Augmentation image_inputs = images / 127.5 - 1 aug = lambda: dataset.augment_image(image_inputs, config.minibatch_size, use_horizontal_flip=True, rotation_rate=0.3, translation_rate=0.2, cutout_size=25, crop_pixels=10) image_inputs = tf.cond(training, aug, lambda: image_inputs) with tf.variable_scope('Network'): # output = deep_cnn_v1(image_inputs_ph, training, config.emb_size, regularizer_rate) output = wideresnet(image_inputs, training, config.emb_size, regularizer_rate=config.regularizer_rate, fmaps=[80, 160, 320, 640], nbof_unit=[4, 4, 4, 4], strides=[2, 2, 2, 2], dropouts=[0., 0., 0., 0.]) # output = deep_cnn_v1(image_inputs, training, config.emb_size, regularizer_rate=config.regularizer_rate, fmaps=[16,32,64,128]) # output = insightface_resnet( # inputs, # training, # emb_size, # regularizer_rate=regularizer_rate, # dropout_rate=0., # fmaps = [16,16,32,64,128], # nbof_unit = [1,1,1,1,1], # strides = [2,2,2,2,2]) emb = tf.nn.l2_normalize(output, axis=1) # Loss network with tf.variable_scope('LossLayers'): # net_logit_layer = layers.dense(emb, config.nbof_labels) # net_logit_layer = losses.cosineface_losses(emb, label_inputs, config.nbof_labels, config.regularizer_rate) net_logit_layer = losses.cosineface_losses(emb, label_inputs, config.minibatch_size, config.regularizer_rate) net_loss_layer = tf.reduce_mean( tf.nn.sparse_softmax_cross_entropy_with_logits( logits=net_logit_layer, labels=label_inputs)) input_emb = tf.compat.v1.placeholder(tf.float32, shape=[None, config.emb_size], name='input_emb') with tf.variable_scope('LossLayers', reuse=True): # logit_layer = layers.dense(input_emb, config.nbof_labels) # logit_layer = losses.cosineface_losses(input_emb, label_inputs, config.nbof_labels, config.regularizer_rate) logit_layer = losses.cosineface_losses(input_emb, label_inputs, config.minibatch_size, config.regularizer_rate) loss_layer = tf.reduce_mean( tf.nn.sparse_softmax_cross_entropy_with_logits( logits=logit_layer, labels=label_inputs)) # Optimizer optimizer_net = tf.compat.v1.train.AdamOptimizer(learning_rate=lr, **config.optimzer_kwargs) optimizer_loss = tf.compat.v1.train.GradientDescentOptimizer( learning_rate=0.01) # optimizer_loss = tf.compat.v1.train.AdamOptimizer(learning_rate=lr*10, **config.optimzer_kwargs) var_list_net = tf.compat.v1.get_collection( tf.compat.v1.GraphKeys.GLOBAL_VARIABLES, scope='Network') var_list_loss = tf.compat.v1.get_collection( tf.compat.v1.GraphKeys.GLOBAL_VARIABLES, scope='LossLayers') assert len(var_list_net) > 0 # Training operations loss_net = tf.compat.v1.placeholder(tf.float32, shape=[], name='loss') training_op_net = optimizer_net.minimize(net_loss_layer, var_list=var_list_net, name='training_op_net') training_op_loss = optimizer_loss.minimize(loss_layer, var_list=var_list_loss, name='training_op_loss') extra_training_op = tf.compat.v1.get_collection( tf.GraphKeys.UPDATE_OPS) # For batch normalization # Initializer init = tf.compat.v1.global_variables_initializer() print('Done: graph built.') # Saver with tf.compat.v1.variable_scope('Saver'): train_acc = tf.reduce_mean( tf.cast( tf.equal(tf.arg_max(tf.nn.softmax(net_logit_layer), 1), label_inputs), tf.float32)) training_accuracy = tf.compat.v1.placeholder( tf.float32, shape=[], name='training_accuracy_placeholder') validation_accuracy = tf.compat.v1.placeholder( tf.float32, shape=[], name='validation_accuracy_placeholder') saver = train.Saver(var_list_net, config.logs_path, summary_dict={ 'loss': loss_net, 'training_accuracy': training_accuracy, 'validation_accuracy': validation_accuracy, 'learning_rate': lr }, restore=None) # Time measurements init_time = time() # Training --> use the configuration file print('Training...') with tf.compat.v1.Session() as sess: # Initialize init.run() # Restore former parameters if config.restore: print('Restoring weight stored in {}'.format(config.restore)) saver.restore(sess, config.restore) # Training ... cur_img = config.start_img max_valid_acc = 0 validation_accuracy_ = 0 while cur_img < config.end_img: # Inputs img_, lab_ = define_batch(training_filenames, training_labels, config.minibatch_size) images_ = [ sess.run(image_decoded, feed_dict={image_jpeg: im}) for im in img_ ] # Change label range: _, counts = np.unique(lab_, return_counts=True) lab_ = np.concatenate([[i] * j for i, j in enumerate(counts)]) # Embeddings emb_ = sess.run(emb, feed_dict={ images: images_, label_inputs: lab_, training: True }) # Training operation for _ in range(100): # Overtrain the last layer sess.run(training_op_loss, feed_dict={ input_emb: emb_, label_inputs: lab_, lr: training_schedule(cur_img) }) # Train the network loss_net_, training_accuracy_, _, _ = sess.run( [ net_loss_layer, train_acc, training_op_net, extra_training_op ], feed_dict={ images: images_, label_inputs: lab_, training: True, lr: training_schedule(cur_img) }) # Validation if cur_img % config.img_per_val == 0: pred_validation = np.empty((0, config.emb_size)) labels_test = np.empty((0)) for i in range(0, len(validation_labels), config.minibatch_size): # images_ = load_images(validation_filenames[i:i+config.minibatch_size]) images_ = [ sess.run(image_decoded, feed_dict={image_jpeg: im}) for im in validation_filenames[i:i + config.minibatch_size] ] feed_dict = {images: images_, training: False} pred_validation = np.append(pred_validation, sess.run(emb, feed_dict), axis=0) labels_test = np.append( labels_test, validation_labels[i:i + config.minibatch_size]) _, _, acc, val, _, far = evaluate.evaluate( pred_validation, labels_test[::2]) validation_accuracy_ = np.mean(acc) # Display information graph_time = time() - init_time hours = int(graph_time // 3600) minutes = int(graph_time // 60) - hours * 60 secondes = graph_time - minutes * 60 - hours * 3600 print( '{:4d} kimgs, {}h {:2d}m {:2.1f}s, lr:{:0.6f}, training: {:2.2f}%, validation: {:2.2f}%' .format(cur_img // 1000, hours, minutes, secondes, training_schedule(cur_img), training_accuracy_ * 100, validation_accuracy_ * 100)) # Save logs if cur_img % config.img_per_summary == 0: feed_dict_saver = { images: images_, label_inputs: lab_, validation_accuracy: validation_accuracy_, training_accuracy: training_accuracy_, training: False, loss_net: loss_net_, lr: training_schedule(cur_img) } saver.save_summary(sess, feed_dict_saver, cur_img) # Save model # if cur_img % config.img_per_save == 0: if max_valid_acc < validation_accuracy_: saver.save_model(sess, cur_img // 1000) max_valid_acc = validation_accuracy_ # Update current image cur_img += config.minibatch_size # Final Saving saver.save_model(sess, 'final') saver.close_summary() print('Done: training')
def train_triplet_from_images(): """ Train the classification or the recognition network mode is in ['classification', 'recognition'] """ # Graph definition: inputs, model, loss, optimizer, initializer print('Graph building...') # Inputs images = tf.compat.v1.placeholder( tf.float32, shape=[None, config.image_size, config.image_size, 3], name='image_inputs') # label_inputs = tf.compat.v1.placeholder(tf.int64, shape=[None,], name='label_inputs') training = tf.placeholder(tf.bool, shape=[], name='training') lr = tf.compat.v1.placeholder(tf.float32, shape=[], name='learning_rate') # Augmentation image_inputs = images / 127.5 - 1 aug = lambda: dataset.augment_image(image_inputs, config.minibatch_size, use_horizontal_flip=True, rotation_rate=0.3, translation_rate=0.2, cutout_size=25, crop_pixels=10) image_inputs = tf.cond(training, aug, lambda: image_inputs) # Network emb, loss, reg_loss = network.recognition(image_inputs, config.emb_size, training, config.regularizer_rate) # Optimizer optimizer = tf.compat.v1.train.AdamOptimizer(learning_rate=lr, **config.optimzer_kwargs) var_list = tf.compat.v1.get_collection( tf.compat.v1.GraphKeys.GLOBAL_VARIABLES, scope=config.network) assert len(var_list) > 0 # Training operations training_op = optimizer.minimize(loss + reg_loss, var_list=var_list, name='training_op') extra_training_op = tf.compat.v1.get_collection( tf.GraphKeys.UPDATE_OPS) # For batch normalization # Initializer init = tf.compat.v1.global_variables_initializer() print('Done: graph built.') # Prepare the dataset training_filenames, training_labels = prepare_data.prepare_data( config.data_path) validation_filenames, validation_labels = prepare_data.prepare_dogfacenet( config.data_test_path) # Saver with tf.compat.v1.variable_scope('Saver'): training_accuracy = losses.triplet_accuracy(emb) # Saver placeholders loss_saver = tf.compat.v1.placeholder(tf.float32, shape=[], name='loss_saver') training_accuracy_saver = tf.compat.v1.placeholder( tf.float32, shape=[], name='training_accuracy_saver') validation_accuracy_saver = tf.compat.v1.placeholder( tf.float32, shape=[], name='validation_accuracy_saver') saver = train.Saver(var_list, config.logs_path, summary_dict={ 'loss': loss_saver, 'training_accuracy': training_accuracy_saver, 'validation_accuracy': validation_accuracy_saver, 'learning_rate': lr }, restore=None) # Time measurements init_time = time() # Training --> use the configuration file print('Training...') with tf.compat.v1.Session() as sess: # Initialize init.run() # Restore former parameters if config.restore: print('Restoring weight stored in {}'.format(config.restore)) saver.restore(sess, config.restore) # Training ... cur_img = config.start_img while cur_img < config.end_img: filenames_, _ = define_triplets_batch( training_filenames, training_labels, nbof_triplet=config.minibatch_size) images_ = load_images(filenames_) feed_dict = { images: images_, training: True, lr: training_schedule(cur_img) } loss_, training_accuracy_, _, _ = sess.run( [loss, training_accuracy, training_op, extra_training_op], feed_dict=feed_dict) # Validation if cur_img % config.img_per_val == 0: pred_validation = np.empty((0, config.emb_size)) labels_test = np.empty((0)) for i in range(0, len(validation_labels), config.minibatch_size): images_ = load_images( validation_filenames[i:i + config.minibatch_size]) feed_dict = {images: images_, training: False} pred_validation = np.append(pred_validation, sess.run(emb, feed_dict), axis=0) labels_test = np.append( labels_test, validation_labels[i:i + config.minibatch_size]) _, _, acc, val, _, far = evaluate.evaluate( pred_validation, labels_test[::2]) validation_accuracy_ = np.mean(acc) feed_dict_saver = { loss_saver: loss_, training_accuracy_saver: training_accuracy_, validation_accuracy_saver: validation_accuracy_, lr: training_schedule(cur_img) } saver.save_summary(sess, feed_dict_saver, cur_img) # Display information graph_time = time() - init_time hours = int(graph_time // 3600) minutes = int(graph_time // 60) - hours * 60 secondes = graph_time - minutes * 60 - hours * 3600 print( '{:4d} kimgs, {}h {:2d}m {:2.1f}s, lr:{:0.6f}, training: {:2.2f}%, validation: {:2.2f}%' .format(cur_img // 1000, hours, minutes, secondes, training_schedule(cur_img), training_accuracy_ * 100, validation_accuracy_ * 100)) # Save logs if cur_img % config.img_per_summary == 0: feed_dict_saver = { loss_saver: loss_, training_accuracy_saver: training_accuracy_, validation_accuracy_saver: validation_accuracy_, lr: training_schedule(cur_img) } saver.save_summary(sess, feed_dict_saver, cur_img) # Save model if cur_img % config.img_per_save == 0: saver.save_model(sess, cur_img // 1000) # Update current image cur_img += config.minibatch_size # Final Saving saver.save_model(sess, 'final') saver.close_summary() print('Done: training')
def train_classification(mode='classification', use_adaptive_loss=False): """ Train the classification or the recognition network mode is in ['classification', 'recognition'] """ assert mode in ['classification', 'recognition'] # Graph definition: inputs, model, loss, optimizer, initializer print('Graph building...') with tf.compat.v1.variable_scope('Dataset'): # Init dataset training_dataset = getattr(dataset, config.data_initilize)(minibatch_size=config.minibatch_size, shuffle=True, **config.data_initilize_kwargs) validation_dataset = getattr(dataset, config.data_test_initilize)(minibatch_size=config.minibatch_size, shuffle=False, repeat=False, **config.data_test_initilize_kwargs) # Inputs handle = tf.placeholder(tf.string, shape=[], name='handle_input') iterator = tf.data.Iterator.from_string_handle(handle, training_dataset.output_types, training_dataset.output_shapes) # image_ins, label_ins = iterator.get_next() image_inputs, label_inputs = iterator.get_next() image_inputs.set_shape([None,config.image_size,config.image_size,3]) label_inputs.set_shape([None,1]) label_inputs = tf.reshape(label_inputs, [-1]) label_inputs = tf.cast(label_inputs, tf.int64) image_inputs = tf.cast(image_inputs, dtype=tf.float32)/127.5 - 1 # Network parameters nbof_labels = config.nbof_labels training = tf.placeholder(tf.bool, shape=[], name='training') regularizer_rate=config.regularizer_rate # Augmentation # if mode=='classification': # aug = lambda: dataset.augment_image( # image_inputs, # config.minibatch_size, # use_horizontal_flip=(mode=='classification'), # rotation_rate =0.3, # translation_rate=0.1, # cutout_size =25, # crop_pixels =10) # image_inputs = tf.cond(training, aug, lambda: image_inputs) # Network if mode=='classification': logit,loss,reg_loss = network.classification(image_inputs, label_inputs, nbof_labels, training, regularizer_rate) else: emb_size = config.emb_size emb,logit,loss,reg_loss = network.recognition(image_inputs, label_inputs, emb_size, nbof_labels, training, use_adaptive_loss=False, regularizer_rate=regularizer_rate) # emb,loss,reg_loss = network.recognition(image_inputs, label_inputs, emb_size, nbof_labels, training, use_adaptive_loss=use_adaptive_loss, regularizer_rate=regularizer_rate) # Optimizer # optimizer = tf.compat.v1.train.MomentumOptimizer(learning_rate=config.learning_rate, momentum=0.9, use_nesterov=True) lr = tf.compat.v1.placeholder(tf.float32, shape=[], name='lr') optimizer = tf.compat.v1.train.AdamOptimizer(learning_rate=lr, **config.optimzer_kwargs) var_list = tf.compat.v1.get_collection(tf.compat.v1.GraphKeys.GLOBAL_VARIABLES, scope=config.network) assert len(var_list)>0 # Training operations training_op = optimizer.minimize(loss+reg_loss, var_list=var_list, name='training_op') extra_training_op = tf.compat.v1.get_collection(tf.GraphKeys.UPDATE_OPS) # For batch normalization # Initializer init = tf.compat.v1.global_variables_initializer() print('Done: graph built.') # Define the minibatch selector with tf.compat.v1.variable_scope('Dataset'): training_iterator = training_dataset.make_one_shot_iterator() validation_iterator = validation_dataset.make_initializable_iterator() # Saver with tf.compat.v1.variable_scope('Saver'): if mode == 'classification': pred = tf.nn.softmax(logit) training_output = tf.reduce_mean(tf.cast(tf.equal(tf.argmax(pred, axis=1), label_inputs), dtype=tf.float32)) else: pred = tf.nn.softmax(logit) training_output = tf.reduce_mean(tf.cast(tf.equal(tf.argmax(pred, axis=1), label_inputs), dtype=tf.float32)) # training_output = losses.general_triplet_accuracy(emb, label_inputs, nbof_labels) training_accuracy = tf.compat.v1.placeholder(tf.float32, shape=[], name='training_accuracy_placeholder') validation_accuracy = tf.compat.v1.placeholder(tf.float32, shape=[], name='validation_accuracy_placeholder') saver = train.Saver(var_list,config.logs_path, summary_dict={'loss':loss, 'reg_loss':reg_loss, 'training_accuracy':training_accuracy, 'validation_accuracy':validation_accuracy, 'lr':lr}, restore=None) # Time measurements init_time = time() train_acc = 0 img_per_train_acc = 0 saved_train_acc = 0 valid_acc = 0 # Training --> use the configuration file print('Training...') with tf.compat.v1.Session() as sess: # Initialize init.run() training_handle = sess.run(training_iterator.string_handle()) validation_handle = sess.run(validation_iterator.string_handle()) # Restore former parameters if config.restore: print('Restoring weight stored in {}'.format(config.restore)) saver.restore(sess, config.restore) # Training ... cur_img = config.start_img while cur_img < config.end_img: # Training operation feed_dict = {handle: training_handle, lr:training_schedule(cur_img), training:True} # feed_dict = {handle: validation_handle, lr:training_schedule(cur_img), training:False} # import matplotlib.pyplot as plt # sess.run(validation_iterator.initializer) # for i in range(1): # imgs, labs = sess.run([image_inputs,label_inputs],feed_dict=feed_dict) # print(labs) # fig = plt.figure() # fig.set_size_inches(1, 1, forward=False) # ax = plt.Axes(fig, [0., 0., 1., 1.]) # ax.set_axis_off() # fig.add_axes(ax) # ax.imshow(imgs[1]/2+0.5) # n = int(np.sqrt(len(imgs))) # for i in range(n): # for j in range(n): # plt.subplot(n,n,i*n+j+1) # plt.imshow(imgs[i*n+j]/2 + 0.5) # plt.show() # return 0 _,_, train_out = sess.run([training_op, extra_training_op, training_output], feed_dict=feed_dict) # Save accuracy train_acc += train_out img_per_train_acc += 1 # Validation if cur_img % config.img_per_val == 0: pred_validation = np.empty((0,nbof_labels) if mode=='classification' else (0,emb_size)) labels_test = np.empty((0)) sess.run(validation_iterator.initializer) while True: try: pred_, labs = sess.run([pred, label_inputs] if mode=='classification' else [emb, label_inputs], feed_dict={handle: validation_handle, training:False}) pred_validation = np.append(pred_validation, pred_, axis=0) labels_test = np.append(labels_test,labs) except tf.errors.OutOfRangeError: break if mode=='classification': valid_acc = np.mean(np.equal(np.argmax(pred_validation, axis=1), labels_test)) else: _, _, acc, val, _, far=evaluate.evaluate(pred_validation, labels_test[::2]) valid_acc = np.mean(acc) feed_dict_saver = {handle: training_handle, validation_accuracy:valid_acc, training_accuracy:saved_train_acc, training:False, lr:training_schedule(cur_img)} saver.save_summary(sess, feed_dict_saver, cur_img) # Display information graph_time = time() - init_time hours = int(graph_time // 3600) minutes = int(graph_time // 60) - hours*60 secondes = graph_time - minutes * 60 - hours * 3600 print('{:4d} kimgs, {}h {:2d}m {:2.1f}s, lr:{:0.6f}, training: {:2.2f}%, validation: {:2.2f}%'.format(cur_img//1000, hours, minutes, secondes, training_schedule(cur_img), saved_train_acc*100, valid_acc*100)) # Save logs if cur_img % config.img_per_summary == 0: assert img_per_train_acc!=0 train_acc /= img_per_train_acc feed_dict_saver = {handle: training_handle, validation_accuracy:valid_acc, training_accuracy:train_acc, training:False, lr:training_schedule(cur_img)} saver.save_summary(sess, feed_dict_saver, cur_img) saved_train_acc = train_acc train_acc = 0 img_per_train_acc = 0 # Save model if cur_img % config.img_per_save == 0: saver.save_model(sess, cur_img//1000) # Update current image cur_img += config.minibatch_size # Final Saving saver.save_model(sess, 'final') saver.close_summary() sess.close() print('Done: training')