def train(args, sess, epoch, image_list, label_list, index_dequeue_op, enqueue_op, image_paths_placeholder, labels_placeholder, learning_rate_placeholder, phase_train_placeholder, batch_size_placeholder, global_step, loss, train_op, summary_op, summary_writer, regularization_losses, learning_rate_schedule_file): batch_number = 0 if args.learning_rate > 0.0: lr = args.learning_rate else: lr = facenet.get_learning_rate_from_file(learning_rate_schedule_file, epoch) index_epoch = sess.run(index_dequeue_op) label_epoch = np.array(label_list)[index_epoch] image_epoch = np.array(image_list)[index_epoch] # Enqueue one epoch of image paths and labels labels_array = np.expand_dims(np.array(label_epoch), 1) image_paths_array = np.expand_dims(np.array(image_epoch), 1) sess.run( enqueue_op, { image_paths_placeholder: image_paths_array, labels_placeholder: labels_array }) # Training loop train_time = 0 while batch_number < args.epoch_size: start_time = time.time() feed_dict = { learning_rate_placeholder: lr, phase_train_placeholder: True, batch_size_placeholder: args.batch_size } if (batch_number % 100 == 0): err, _, step, reg_loss, summary_str = sess.run([ loss, train_op, global_step, regularization_losses, summary_op ], feed_dict=feed_dict) summary_writer.add_summary(summary_str, global_step=step) else: err, _, step, reg_loss = sess.run( [loss, train_op, global_step, regularization_losses], feed_dict=feed_dict) duration = time.time() - start_time print('Epoch: [%d][%d/%d]\tTime %.3f\tLoss %2.3f\tRegLoss %2.3f' % (epoch, batch_number + 1, args.epoch_size, duration, err, np.sum(reg_loss))) batch_number += 1 train_time += duration # Add validation loss and accuracy to summary summary = tf.Summary() #pylint: disable=maybe-no-member summary.value.add(tag='time/total', simple_value=train_time) summary_writer.add_summary(summary, step) return step
def train(args, sess, epoch, learning_rate_placeholder, global_step, loss, train_op, summary_op, summary_writer, regularization_losses, learning_rate_schedule_file): batch_number = 0 if args.learning_rate > 0.0: lr = args.learning_rate else: lr = facenet.get_learning_rate_from_file(learning_rate_schedule_file, epoch) # Training loop while batch_number < args.epoch_size: train_time = 0 i = 0 while batch_number < args.epoch_size: start_time = time.time() feed_dict = {learning_rate_placeholder: lr} err, _, step, reg_loss = sess.run([loss, train_op, global_step, regularization_losses], feed_dict=feed_dict) if (batch_number % 100 == 0): summary_str, step = sess.run([summary_op, global_step], feed_dict=feed_dict) summary_writer.add_summary(summary_str, global_step=step) duration = time.time() - start_time print('Epoch: [%d][%d/%d]\tTime %.3f\tLoss %2.3f\tRegLoss %2.3f' % (epoch, batch_number+1, args.epoch_size, duration, err, np.sum(reg_loss))) batch_number += 1 i += 1 train_time += duration # Add validation loss and accuracy to summary summary = tf.Summary() #pylint: disable=maybe-no-member summary.value.add(tag='time/total', simple_value=train_time) summary_writer.add_summary(summary, step) return step
def train(args, sess, epoch, phase_train_placeholder, learning_rate_placeholder, global_step, loss, train_op, summary_op, summary_writer, regularization_losses, learning_rate_schedule_file): batch_number = 0 if args.learning_rate>0.0: lr = args.learning_rate else: lr = facenet.get_learning_rate_from_file(learning_rate_schedule_file, epoch) while batch_number < args.epoch_size: # Perform training on the selected triplets train_time = 0 i = 0 while batch_number < args.epoch_size: start_time = time.time() feed_dict = {phase_train_placeholder: True, learning_rate_placeholder: lr} err, _, step, reg_loss = sess.run([loss, train_op, global_step, regularization_losses], feed_dict=feed_dict) if (batch_number % 100 == 0): summary_str, step = sess.run([summary_op, global_step], feed_dict=feed_dict) summary_writer.add_summary(summary_str, global_step=step) duration = time.time() - start_time print('Epoch: [%d][%d/%d]\tTime %.3f\tLoss %2.3f\tRegLoss %2.3f' % (epoch, batch_number+1, args.epoch_size, duration, err, np.sum(reg_loss))) batch_number += 1 i += 1 train_time += duration # Add validation loss and accuracy to summary summary = tf.Summary() #pylint: disable=maybe-no-member summary.value.add(tag='time/total', simple_value=train_time) summary_writer.add_summary(summary, step) return step
def train(args, sess, epoch, image_list, label_list, index_dequeue_op, enqueue_op, image_paths_placeholder, labels_placeholder, learning_rate_placeholder, phase_train_placeholder, batch_size_placeholder, control_placeholder, step, loss, train_op, summary_op, summary_writer, reg_losses, learning_rate_schedule_file, stat, cross_entropy_mean, accuracy, learning_rate, prelogits, random_rotate, random_crop, random_flip, prelogits_norm, prelogits_hist_max, use_fixed_image_standardization): batch_number = 0 if args.learning_rate>0.0: lr = args.learning_rate else: lr = facenet.get_learning_rate_from_file(learning_rate_schedule_file, epoch) if lr<=0: return False index_epoch = sess.run(index_dequeue_op) label_epoch = np.array(label_list)[index_epoch] image_epoch = np.array(image_list)[index_epoch] # Enqueue one epoch of image paths and labels labels_array = np.expand_dims(np.array(label_epoch),1) image_paths_array = np.expand_dims(np.array(image_epoch),1) control_value = facenet.RANDOM_ROTATE * random_rotate + facenet.RANDOM_CROP * random_crop + facenet.RANDOM_FLIP * random_flip + facenet.FIXED_STANDARDIZATION * use_fixed_image_standardization control_array = np.ones_like(labels_array) * control_value sess.run(enqueue_op, {image_paths_placeholder: image_paths_array, labels_placeholder: labels_array, control_placeholder: control_array}) # Training loop train_time = 0 while batch_number < args.epoch_size: start_time = time.time() feed_dict = {learning_rate_placeholder: lr, phase_train_placeholder:True, batch_size_placeholder:args.batch_size} tensor_list = [loss, train_op, step, reg_losses, prelogits, cross_entropy_mean, learning_rate, prelogits_norm, accuracy] if batch_number % 100 == 0: loss_, _, step_, reg_losses_, prelogits_, cross_entropy_mean_, lr_, prelogits_norm_, accuracy_, summary_str = sess.run(tensor_list + [summary_op], feed_dict=feed_dict) summary_writer.add_summary(summary_str, global_step=step_) else: loss_, _, step_, reg_losses_, prelogits_, cross_entropy_mean_, lr_, prelogits_norm_, accuracy_ = sess.run(tensor_list, feed_dict=feed_dict) duration = time.time() - start_time stat['loss'][step_-1] = loss_ #stat['center_loss'][step_-1] = center_loss_ stat['reg_loss'][step_-1] = np.sum(reg_losses_) stat['xent_loss'][step_-1] = cross_entropy_mean_ stat['prelogits_norm'][step_-1] = prelogits_norm_ stat['learning_rate'][epoch-1] = lr_ stat['accuracy'][step_-1] = accuracy_ stat['prelogits_hist'][epoch-1,:] += np.histogram(np.minimum(np.abs(prelogits_), prelogits_hist_max), bins=1000, range=(0.0, prelogits_hist_max))[0] duration = time.time() - start_time print('Epoch: [%d][%d/%d]\tTime %.3f\tLoss %2.3f\tXent %2.3f\tRegLoss %2.3f\tAccuracy %2.3f\tLr %2.5f' % (epoch, batch_number+1, args.epoch_size, duration, loss_, cross_entropy_mean_, np.sum(reg_losses_), accuracy_, lr_)) batch_number += 1 train_time += duration # Add validation loss and accuracy to summary summary = tf.Summary() #pylint: disable=maybe-no-member summary.value.add(tag='time/total', simple_value=train_time) summary_writer.add_summary(summary, global_step=step_) return True
def train(args, sess, epoch, image_list, label_list, index_dequeue_op, enqueue_op, image_paths_placeholder, labels_placeholder, learning_rate_placeholder, phase_train_placeholder, batch_size_placeholder, control_placeholder, step, loss, train_op, summary_op, summary_writer, reg_losses, learning_rate_schedule_file, stat, cross_entropy_mean, accuracy, learning_rate, prelogits, prelogits_center_loss, random_rotate, random_crop, random_flip, prelogits_norm, prelogits_hist_max, use_fixed_image_standardization): batch_number = 0 if args.learning_rate>0.0: lr = args.learning_rate else: lr = facenet.get_learning_rate_from_file(learning_rate_schedule_file, epoch) if lr<=0: return False index_epoch = sess.run(index_dequeue_op) label_epoch = np.array(label_list)[index_epoch] image_epoch = np.array(image_list)[index_epoch] # Enqueue one epoch of image paths and labels labels_array = np.expand_dims(np.array(label_epoch),1) image_paths_array = np.expand_dims(np.array(image_epoch),1) control_value = facenet.RANDOM_ROTATE * random_rotate + facenet.RANDOM_CROP * random_crop + facenet.RANDOM_FLIP * random_flip + facenet.FIXED_STANDARDIZATION * use_fixed_image_standardization control_array = np.ones_like(labels_array) * control_value sess.run(enqueue_op, {image_paths_placeholder: image_paths_array, labels_placeholder: labels_array, control_placeholder: control_array}) # Training loop train_time = 0 while batch_number < args.epoch_size: start_time = time.time() feed_dict = {learning_rate_placeholder: lr, phase_train_placeholder:True, batch_size_placeholder:args.batch_size} tensor_list = [loss, train_op, step, reg_losses, prelogits, cross_entropy_mean, learning_rate, prelogits_norm, accuracy, prelogits_center_loss] if batch_number % 100 == 0: loss_, _, step_, reg_losses_, prelogits_, cross_entropy_mean_, lr_, prelogits_norm_, accuracy_, center_loss_, summary_str = sess.run(tensor_list + [summary_op], feed_dict=feed_dict) summary_writer.add_summary(summary_str, global_step=step_) else: loss_, _, step_, reg_losses_, prelogits_, cross_entropy_mean_, lr_, prelogits_norm_, accuracy_, center_loss_ = sess.run(tensor_list, feed_dict=feed_dict) duration = time.time() - start_time stat['loss'][step_-1] = loss_ stat['center_loss'][step_-1] = center_loss_ stat['reg_loss'][step_-1] = np.sum(reg_losses_) stat['xent_loss'][step_-1] = cross_entropy_mean_ stat['prelogits_norm'][step_-1] = prelogits_norm_ stat['learning_rate'][epoch-1] = lr_ stat['accuracy'][step_-1] = accuracy_ stat['prelogits_hist'][epoch-1,:] += np.histogram(np.minimum(np.abs(prelogits_), prelogits_hist_max), bins=1000, range=(0.0, prelogits_hist_max))[0] duration = time.time() - start_time print('Epoch: [%d][%d/%d]\tTime %.3f\tLoss %2.3f\tXent %2.3f\tRegLoss %2.3f\tAccuracy %2.3f\tLr %2.5f\tCl %2.3f' % (epoch, batch_number+1, args.epoch_size, duration, loss_, cross_entropy_mean_, np.sum(reg_losses_), accuracy_, lr_, center_loss_)) batch_number += 1 train_time += duration # Add validation loss and accuracy to summary summary = tf.Summary() #pylint: disable=maybe-no-member summary.value.add(tag='time/total', simple_value=train_time) summary_writer.add_summary(summary, global_step=step_) return True
def test(confusion, accuracy_, args, sess, epoch, image_list, label_list, index_dequeue_op, enqueue_op, image_paths_placeholder, labels_placeholder, learning_rate_placeholder, phase_train_placeholder, batch_size_placeholder, global_step, loss, train_op, summary_op, summary_writer, regularization_losses, learning_rate_schedule_file): # batch_number = 0 if args.learning_rate > 0.0: lr = args.learning_rate else: lr = facenet.get_learning_rate_from_file(learning_rate_schedule_file, epoch) # index_epoch = sess.run(index_dequeue_op) # label_epoch = np.array(label_list)[index_epoch] # image_epoch = np.array(image_list)[index_epoch] # Enqueue one epoch of image paths and labels labels_array = np.expand_dims(np.array(label_list), 1) image_paths_array = np.expand_dims(np.array(image_list), 1) #print(labels_array) sess.run( enqueue_op, { image_paths_placeholder: image_paths_array, labels_placeholder: labels_array }) #print(np.shape(labels_array)) # # Training loop # # train_time = 0 # # while batch_number < args.epoch_size: # # start_time = time.time() feed_dict = { learning_rate_placeholder: lr, phase_train_placeholder: False, batch_size_placeholder: args.lfw_batch_size } # # if (batch_number % 100 == 0): # # err, _, step, reg_loss, summary_str,acc = sess.run([loss, train_op, global_step, regularization_losses, summary_op, accuracy], feed_dict=feed_dict) # # summary_writer.add_summary(summary_str, global_step=step) # # else: # # err, _, step, reg_loss = sess.run([loss, train_op, global_step, regularization_losses], feed_dict=feed_dict) # # duration = time.time() - start_time # # print('Epoch: [%d][%d/%d]\tTime %.3f\tLoss %2.3f\tRegLoss %2.3f' % # # (epoch, batch_number+1, args.epoch_size, duration, err, np.sum(reg_loss))) acc, m = sess.run([accuracy_, confusion], feed_dict=feed_dict) print('accuracy', acc) print('confusion_m', m) # # batch_number += 1 # # train_time += duration # # Add validation loss and accuracy to summary # summary = tf.Summary() # #pylint: disable=maybe-no-member # summary.value.add(tag='time/total', simple_value=train_time) # summary_writer.add_summary(summary, step) return acc, m
def train(args, sess, epoch, image_list, label_list, enqueue_op, image_paths_placeholder, labels_placeholder, learning_rate_placeholder, phase_train_placeholder, batch_size_placeholder, global_step, loss, train_op, summary_op, summary_writer, regularization_losses, learning_rate_schedule_file): batch_number = 0 if args.learning_rate>0.0: lr = args.learning_rate else: lr = facenet.get_learning_rate_from_file(learning_rate_schedule_file, epoch) nrof_examples = len(image_list) nrof_examples_per_epoch = args.epoch_size*args.batch_size j = epoch*nrof_examples_per_epoch % nrof_examples if j+args.epoch_size*args.batch_size<=nrof_examples: label_epoch = label_list[j:j+nrof_examples_per_epoch] image_epoch = image_list[j:j+nrof_examples_per_epoch] else: label_epoch = label_list[j:nrof_examples] image_epoch = image_list[j:nrof_examples] image_list, label_list = facenet.shuffle_examples(image_list, label_list) label_epoch += label_list[0:nrof_examples_per_epoch-(nrof_examples-j)] image_epoch += image_list[0:nrof_examples_per_epoch-(nrof_examples-j)] # Enqueue one epoch of image paths and labels labels_array = np.expand_dims(np.array(label_epoch),1) image_paths_array = np.expand_dims(np.array(image_epoch),1) ## Really Enque the data into the queue !!!! Here is the start of the whole process of the training, i.e the start of the training graph ######## sess.run(enqueue_op, {image_paths_placeholder: image_paths_array, labels_placeholder: labels_array}) # Training loop train_time = 0 while batch_number < args.epoch_size: start_time = time.time() feed_dict = {learning_rate_placeholder: lr, phase_train_placeholder:True, batch_size_placeholder:args.batch_size} if (batch_number % 100 == 0): err, _, step, reg_loss, summary_str = sess.run([loss, train_op, global_step, regularization_losses, summary_op], feed_dict=feed_dict) summary_writer.add_summary(summary_str, global_step=step) else: err, _, step, reg_loss = sess.run([loss, train_op, global_step, regularization_losses], feed_dict=feed_dict) duration = time.time() - start_time print('Epoch: [%d][%d/%d]\tTime %.3f\tLoss %2.3f\tRegLoss %2.3f' % (epoch, batch_number+1, args.epoch_size, duration, err, np.sum(reg_loss))) batch_number += 1 train_time += duration # Add validation loss and accuracy to summary summary = tf.Summary() #pylint: disable=maybe-no-member summary.value.add(tag='time/total', simple_value=train_time) summary_writer.add_summary(summary, step) return step
def train(args, sess, epoch, image_list, label_list, index_dequeue_op, enqueue_op, image_paths_placeholder, labels_placeholder, learning_rate_placeholder, phase_train_placeholder, batch_size_placeholder, global_step, loss, train_op, summary_op, summary_writer, regularization_losses, learning_rate_schedule_file): batch_number = 0 if args.learning_rate > 0.0: lr = args.learning_rate else: lr = facenet.get_learning_rate_from_file(learning_rate_schedule_file, epoch) # 每次训练时,出列一次,出列的个数是args.batch_size*args.epoch_size,,即一个epoch中图像的数量 index_epoch = sess.run(index_dequeue_op) label_epoch = np.array(label_list)[index_epoch] image_epoch = np.array(image_list)[index_epoch] # Enqueue one epoch of image paths and labels labels_array = np.expand_dims(np.array(label_epoch), 1) image_paths_array = np.expand_dims(np.array(image_epoch), 1) # 将对应的文件路径以及标签入列 sess.run(enqueue_op, {image_paths_placeholder: image_paths_array, labels_placeholder: labels_array}) # Training loop train_time = 0 while batch_number < args.epoch_size: # print('ok') start_time = time.time() feed_dict = {learning_rate_placeholder: lr, phase_train_placeholder: True, batch_size_placeholder: args.batch_size} # 每100次,将结果保存到log中去 # 然后运行一次,是一次运行:求loss,根据loss,运行train_op来对参数进行优化 # 计算REGULARIZATION_LOSSES只是为了打印出来查看正则损失的值,而实际整个训练过程这个值已经包含在total loss中了 if (batch_number % 100 == 0): err, _, step, reg_loss, summary_str = sess.run( [loss, train_op, global_step, regularization_losses, summary_op], feed_dict=feed_dict) summary_writer.add_summary(summary_str, global_step=step) else: err, _, step, reg_loss = sess.run([loss, train_op, global_step, regularization_losses], feed_dict=feed_dict) duration = time.time() - start_time print('Epoch: [%d][%d/%d]\tTime %.3f\tLoss %2.3f\tRegLoss %2.3f' % (epoch, batch_number + 1, args.epoch_size, duration, err, np.sum(reg_loss))) batch_number += 1 train_time += duration # Add validation loss and accuracy to summary summary = tf.Summary() # pylint: disable=maybe-no-member summary.value.add(tag='time/total', simple_value=train_time) summary_writer.add_summary(summary, step) return step
def train(args, sess, epoch, learning_rate_placeholder, phase_train_placeholder, global_step, loss, train_op, summary_op, summary_writer, regularization_losses, accuracy, learning_rate): batch_number = 0 if args.learning_rate > 0.0: lr = args.learning_rate else: lr = facenet.get_learning_rate_from_file( args.learning_rate_schedule_file, epoch) print('training a epoch...') # Training loop train_time = 0 while batch_number < args.epoch_size: start_time = time.time() feed_dict = { learning_rate_placeholder: lr, phase_train_placeholder: True } if (batch_number % 100 == 0): err, _, step, reg_loss, accuracy_, lr_, summary_str = sess.run( [ loss, train_op, global_step, regularization_losses, accuracy, learning_rate, summary_op ], feed_dict=feed_dict) summary_writer.add_summary(summary_str, global_step=step) else: err, _, step, reg_loss, accuracy_, lr_ = sess.run( [ loss, train_op, global_step, regularization_losses, accuracy, learning_rate ], feed_dict=feed_dict) duration = time.time() - start_time print('Epoch:[%d][%d/%d]\tTime:%.2f Loss:%2.3f Lr:%2.3f Acc:%2.3f' % (epoch, batch_number + 1, args.epoch_size, duration, err, lr_, accuracy_)) batch_number += 1 train_time += duration # Add validation loss and accuracy to summary summary = tf.Summary() #pylint: disable=maybe-no-member summary.value.add(tag='time/total', simple_value=train_time) summary_writer.add_summary(summary, step) return step
def train(args, sess, epoch, image_list, label_list, index_dequeue_op, enqueue_op, image_paths_placeholder, labels_placeholder, learning_rate_placeholder, phase_train_placeholder, batch_size_placeholder, global_step, loss, train_op, summary_op, summary_writer, regularization_losses, learning_rate_schedule_file): batch_number = 0 if args.learning_rate>0.0: lr = args.learning_rate else: lr = facenet.get_learning_rate_from_file(learning_rate_schedule_file, epoch) index_epoch = sess.run(index_dequeue_op) label_epoch = np.array(label_list)[index_epoch] image_epoch = np.array(image_list)[index_epoch] # Enqueue one epoch of image paths and labels labels_array = np.expand_dims(np.array(label_epoch),1) image_paths_array = np.expand_dims(np.array(image_epoch),1) sess.run(enqueue_op, {image_paths_placeholder: image_paths_array, labels_placeholder: labels_array}) # Training loop train_time = 0 while batch_number < args.epoch_size: start_time = time.time() feed_dict = {learning_rate_placeholder: lr, phase_train_placeholder:True, batch_size_placeholder:args.batch_size} if (batch_number % 100 == 0): err, _, step, reg_loss, summary_str = sess.run([loss, train_op, global_step, regularization_losses, summary_op], feed_dict=feed_dict) summary_writer.add_summary(summary_str, global_step=step) else: err, _, step, reg_loss = sess.run([loss, train_op, global_step, regularization_losses], feed_dict=feed_dict) duration = time.time() - start_time print('Epoch: [%d][%d/%d]\tTime %.3f\tLoss %2.3f\tRegLoss %2.3f' % (epoch, batch_number+1, args.epoch_size, duration, err, np.sum(reg_loss))) batch_number += 1 train_time += duration # Add validation loss and accuracy to summary summary = tf.Summary() #pylint: disable=maybe-no-member summary.value.add(tag='time/total', simple_value=train_time) summary_writer.add_summary(summary, step) return step
def train(args, sess, epoch, learning_rate_placeholder, phase_train_placeholder, global_step, loss, train_op, summary_op, summary_writer, learning_rate_schedule_file): batch_number = 0 # learning rate 设置 if args.learning_rate > 0.0: lr = args.learning_rate else: # learning rate 小于 0 的话,就从 文件中获取对应的 learning rate lr = facenet.get_learning_rate_from_file(learning_rate_schedule_file, epoch) # 在 batch_number 小于 epoch_size(实际上是 每个 epoch 中 btach 的总数)时,进行以下操作 while batch_number < args.epoch_size: start_time = time.time() print('Running forward pass on sampled images: ', end='') # feed 进行传 learning_rate feed_dict = { learning_rate_placeholder: lr, phase_train_placeholder: True } start_time = time.time() # sess.run() total_err, reg_err, _, step = sess.run( [loss['total_loss'], loss['total_reg'], train_op, global_step], feed_dict=feed_dict) duration = time.time() - start_time print( 'Epoch: [%d][%d/%d]\tTime %.3f\tTotal Loss %2.3f\tReg Loss %2.3f, lr %2.5f' % (epoch, batch_number + 1, args.epoch_size, duration, total_err, reg_err, lr)) # 训练完成一个 batch 之后,进行 batch_number + 1,进入下一个 batch 的运行 batch_number += 1 return step
def train(args, sess, epoch, image_list, label_list, index_dequeue_op, enqueue_op, image_paths_placeholder, labels_placeholder, learning_rate_placeholder, phase_train_placeholder, batch_size_placeholder, control_placeholder, step, loss, train_op, summary_op, summary_writer, reg_losses, learning_rate_schedule_file, stat, cross_entropy_mean, accuracy, learning_rate, prelogits, prelogits_center_loss, random_rotate, random_crop, random_flip, prelogits_norm, prelogits_hist_max, use_fixed_image_standardization): batch_number = 0 if args.learning_rate > 0.0: lr = args.learning_rate else: lr = facenet.get_learning_rate_from_file(learning_rate_schedule_file, epoch) if lr <= 0: return False index_epoch = sess.run(index_dequeue_op) image_epoch = np.array(image_list)[index_epoch] label_epoch = np.array(label_list)[index_epoch] labels_array = np.expand_dims(np.array(label_epoch), 1) image_path_array = np.expand_dims(np.array(image_epoch), 1) control_value = facenet.RANDOM_ROTATE * random_rotate + facenet.RANDOM_CROP * random_crop + \ facenet.RANDOM_FLIP * random_flip + facenet.FIXED_STANDARDIZATION * use_fixed_image_standardization control_array = np.ones_like(labels_array) * control_value sess.run(enqueue_op, {image_paths_placeholder: image_path_array, labels_placeholder: labels_array, control_placeholder: control_array}) train_time = 0 while batch_number < args.epoch_size: start_time = time.time() feed_dict = {learning_rate_placeholder: lr, phase_train_placeholder: True, batch_size_placeholder: args.batch_size} tensor_list = [loss, train_op, step, reg_losses, prelogits, cross_entropy_mean, leraning_rate, prelogits_norm, accuracy, prelogits_center_loss] if batch_number % 100 == 0: _loss, _, _step, _reg_losses, _prelogits, _cross_entropy_mean, _lr, _prelogits_norm, _accuracy, \ _center_loss, summary_str = sess.run(tensor_list + [summary_op], feed_dict=feed_dict) else: _loss, _, _step, _reg_losses, _prelogits, _cross_entropy_mean, _lr, _prelogits_norm, _accuracy, \ _center_loss = sess.run(tensor_list, feed_dict=feed_dict) duration = time.time() - start_time stat['loss'][_step - 1] = _loss stat['center_loss'][_step - 1] = _center_loss stat['reg_loss'][_step - 1] = np.sum(_reg_losses) stat['xent_loss'][_step - 1] = _cross_entropy_mean stat['prelogits_norm'][_step - 1] = _prelogits_norm stat['learning_rate'][epoch - 1] = _lr stat['accuracy'][_step - 1] = _accuracy stat['prelogits_hist'][epoch - 1,:] += np.histogram(np.minimum(np.abs(_prelogits), prelogits_hist_max), bins=1000, range=(0.0, prelogits_hist_max))[0] duration = time.time() - start_time print('Epoch: [%d][%d/%d]\tTime %.3f\tLoss %2.3f\tXent %2.3f\tRegLoss %2.3f\tAccuracy %2.3f\tLr %2.5f\tCl %2.3f' % (epoch, batch_number + 1, args.epoch_size, duration, _loss, _cross_entropy_mean, np.sum(_reg_losses), _accuracy, _lr, _center_loss)) batch_number += 1 train_time += duration summary = tf.Summary() summary.value.add(tag='time/total', simple_value=train_time) summary_writer.add_summary(summary, global_step=_step) return True
def train(args, sess, epoch, image_list, label_list, index_dequeue_op, enqueue_op, image_paths_placeholder, labels_placeholder, learning_rate_placeholder, phase_train_placeholder, batch_size_placeholder, control_placeholder, step, loss, train_op, summary_op, summary_writer, reg_losses, learning_rate_schedule_file, stat, cross_entropy_mean, accuracy, learning_rate, prelogits, prelogits_center_loss, random_rotate, random_crop, random_flip, prelogits_norm, prelogits_hist_max, use_fixed_image_standardization): import imgaug import imgaug.augmenters as iaa batch_number = 0 augmentation = iaa.Sequential([iaa.Rotate((-170, 170))]) if args.learning_rate > 0.0: lr = args.learning_rate else: lr = facenet.get_learning_rate_from_file(learning_rate_schedule_file, epoch) if lr <= 0: return False index_epoch = sess.run(index_dequeue_op) label_epoch = np.array(label_list)[index_epoch] image_epoch = np.array(image_list)[index_epoch] # Enqueue one epoch of image paths and labels labels_array = np.expand_dims(np.array(label_epoch), 1) image_paths_array = np.expand_dims(np.array(image_epoch), 1) images_array = [] for p in image_paths_array: imgs = [] for path in p: image = np.array(skimage.io.imread(path)) if len(image.shape) == 2: image = np.stack([image, image, image], axis=2) image = image[:, :, 0:3] w, h = image.shape r = math.ceil((w**2 + h**2)**0.5) pw = math.ceil((r - w) / 2) ph = math.ceil((r - h) / 2) image = np.pad(image, ((pw, pw), (ph, ph), (0, 0))) m = path[:-4] + '.npy' with open(m, 'rb') as file: mask = np.load(file) mask = np.pad(mask, ((pw, pw), (ph, ph))) # Augmenters that are safe to apply to masks # Some, such as Affine, have settings that make them unsafe, so always # test your augmentation on masks MASK_AUGMENTERS = [ "Sequential", "SomeOf", "OneOf", "Sometimes", "Fliplr", "Flipud", "CropAndPad", "Affine", "PiecewiseAffine" ] def hook(images, augmenter, parents, default): """Determines which augmenters to apply to masks.""" return augmenter.__class__.__name__ in MASK_AUGMENTERS # Store shapes before augmentation to compare image_shape = image.shape mask_shape = mask.shape # Make augmenters deterministic to apply similarly to images and masks det = augmentation.to_deterministic() image = det.augment_image(image) mask = det.augment_image(mask) # Verify that shapes didn't change assert image.shape == image_shape, "Augmentation shouldn't change image size" assert mask.shape == mask_shape, "Augmentation shouldn't change mask size" horizontal_indicies = np.where(np.any(mask, axis=0))[0] vertical_indicies = np.where(np.any(mask, axis=1))[0] x1, x2 = horizontal_indicies[[0, -1]] y1, y2 = vertical_indicies[[0, -1]] x2 += 1 y2 += 1 mask_out = image * np.stack([mask, mask, mask], axis=2) out = mask_out[y1:y2, x1:x2] out = cv2.resize(out, dsize=(args.image_size, args.image_size)) #cv2.imwrite('verify.png', out) imgs.append(out) images_array.append(imgs) images_array = np.array(images_array) control_value = facenet.RANDOM_ROTATE * random_rotate + facenet.RANDOM_CROP * random_crop + \ facenet.RANDOM_FLIP * random_flip + facenet.FIXED_STANDARDIZATION * use_fixed_image_standardization control_array = np.ones_like(labels_array) * control_value sess.run( enqueue_op, { # image_paths_placeholder: image_paths_array, image_paths_placeholder: images_array, labels_placeholder: labels_array, control_placeholder: control_array }) # Training loop train_time = 0 while batch_number < args.epoch_size: start_time = time.time() feed_dict = { learning_rate_placeholder: lr, phase_train_placeholder: True, batch_size_placeholder: args.batch_size } tensor_list = [ loss, train_op, step, reg_losses, prelogits, cross_entropy_mean, learning_rate, prelogits_norm, accuracy, prelogits_center_loss ] if batch_number % 100 == 0: loss_, _, step_, reg_losses_, prelogits_, cross_entropy_mean_, lr_, prelogits_norm_, accuracy_, center_loss_, summary_str = sess.run( tensor_list + [summary_op], feed_dict=feed_dict) summary_writer.add_summary(summary_str, global_step=step_) else: loss_, _, step_, reg_losses_, prelogits_, cross_entropy_mean_, lr_, prelogits_norm_, accuracy_, center_loss_ = sess.run( tensor_list, feed_dict=feed_dict) duration = time.time() - start_time stat['loss'][step_ - 1] = loss_ stat['center_loss'][step_ - 1] = center_loss_ stat['reg_loss'][step_ - 1] = np.sum(reg_losses_) stat['xent_loss'][step_ - 1] = cross_entropy_mean_ stat['prelogits_norm'][step_ - 1] = prelogits_norm_ stat['learning_rate'][epoch - 1] = lr_ stat['accuracy'][step_ - 1] = accuracy_ stat['prelogits_hist'][epoch - 1, :] += np.histogram( np.minimum(np.abs(prelogits_), prelogits_hist_max), bins=1000, range=(0.0, prelogits_hist_max))[0] duration = time.time() - start_time print( 'Epoch: [%d][%d/%d]\tTime %.3f\tLoss %2.3f\tXent %2.3f\tRegLoss %2.3f\tAccuracy %2.3f\tLr %2.5f\tCl %2.3f' % (epoch, batch_number + 1, args.epoch_size, duration, loss_, cross_entropy_mean_, np.sum(reg_losses_), accuracy_, lr_, center_loss_)) batch_number += 1 train_time += duration # Add validation loss and accuracy to summary summary = tf.compat.v1.Summary() #pylint: disable=maybe-no-member summary.value.add(tag='time/total', simple_value=train_time) summary_writer.add_summary(summary, global_step=step_) return True
def train(args, sess, epoch, image_list, label_list, enqueue_op, image_paths_placeholder, labels_placeholder, learning_rate_placeholder, phase_train_placeholder, batch_size_placeholder, global_step, loss, train_op, summary_op, summary_writer, regularization_losses, learning_rate_schedule_file, class_triplet_loss, acc, val, far, cross_entropy_mean, dist_within_center, dist_inter_centers_sum_all, nrof_centers_batch, image_batch, label_batch, min_num_classes_batch, min_num_imgs_class, index_dequeue_op, train_acc, logits, correct_prediction, log_dir): batch_number = 0 if args.learning_rate > 0.0: lr = args.learning_rate else: lr = facenet.get_learning_rate_from_file(learning_rate_schedule_file, epoch) ## Select the images for a epoch in which each batch includes at least two different classes ################################################################################ # min_num_classes_batch = 3 # min_num_imgs_class = 10 #image_epoch, label_epoch = facenet.select_batch_images(image_list, label_list, epoch, args.epoch_size, args.batch_size, min_num_classes_batch, min_num_imgs_class) ###old method for fetch images and labels for each epoch, in this way the images and labes are not shuffled maye repeatly fetch the same image and labels in iterations ### # nrof_examples = len(image_list) # nrof_examples_per_epoch = args.epoch_size * args.batch_size # j = epoch * nrof_examples_per_epoch % nrof_examples # if j + args.epoch_size * args.batch_size <= nrof_examples: # label_epoch = label_list[j:j + nrof_examples_per_epoch] # image_epoch = image_list[j:j + nrof_examples_per_epoch] # else: # label_epoch = label_list[j:nrof_examples] # image_epoch = image_list[j:nrof_examples] # image_list, label_list = facenet.shuffle_examples(image_list, label_list) # label_epoch += label_list[0:nrof_examples_per_epoch - (nrof_examples - j)] # image_epoch += image_list[0:nrof_examples_per_epoch - (nrof_examples - j)] ############################################################################################################################################################################# ###################### shuffling the image and labels for each training epoch with the shuffle fetching index for each eapoch ############################################### index_epoch = sess.run(index_dequeue_op) label_epoch = np.array(label_list)[index_epoch] image_epoch = np.array(image_list)[index_epoch] print( 'length of index_epoch: %d=======================================================\n' % len(index_epoch)) #for idx in index_epoch: #print('%d,'%idx) # Enqueue one epoch of image paths and labels print('Enqueue epoch images...\n') labels_array = np.expand_dims(np.array(label_epoch), 1) image_paths_array = np.expand_dims(np.array(image_epoch), 1) ## Really Enque the data into the queue !!!! Here is the start of the whole process of the training, i.e the start of the training graph ######## sess.run( enqueue_op, { image_paths_placeholder: image_paths_array, labels_placeholder: labels_array }) # Training loop print('Begin trainning....\n') train_time = 0 with open(os.path.join(log_dir, 'Training_result.txt'), 'at') as f: while batch_number < args.epoch_size: start_time = time.time() feed_dict = { learning_rate_placeholder: lr, phase_train_placeholder: True, batch_size_placeholder: args.batch_size } if (batch_number % 100 == 0): # err, center_inter_triplet_loss_, _, step, reg_loss, summary_str, loss_center_, loss_inter_centers_, _ = sess.run([loss, class_triplet_loss, train_op, global_step, regularization_losses, summary_op], feed_dict=feed_dict) # summary_writer.add_summary(summary_str, global_step=step) totalloss, class_triplet_loss_, dist_within_center_, dist_inter_centers_sum_all_, _, step, reg_loss, crossentropyloss, nrof_centers_batch_, image_batch_, label_batch_, train_acc_, logits_, summary_str = sess.run( [ loss, class_triplet_loss, dist_within_center, dist_inter_centers_sum_all, train_op, global_step, regularization_losses, cross_entropy_mean, nrof_centers_batch, image_batch, label_batch, train_acc, logits, summary_op ], feed_dict=feed_dict) summary_writer.add_summary(summary_str, global_step=step) else: totalloss, class_triplet_loss_, dist_within_center_, dist_inter_centers_sum_all_, _, step, reg_loss, crossentropyloss, nrof_centers_batch_, image_batch_, label_batch_, train_acc_, logits_ = sess.run( [ loss, class_triplet_loss, dist_within_center, dist_inter_centers_sum_all, train_op, global_step, regularization_losses, cross_entropy_mean, nrof_centers_batch, image_batch, label_batch, train_acc, logits ], feed_dict=feed_dict) duration = time.time() - start_time # ####################### # print(label_batch_) # raw_input('wait a key...') # ####################### Regloss = np.sum(reg_loss) crossEntropyLoss = crossentropyloss dist_within_center_mean = dist_within_center_ / (args.batch_size) dist_inter_centers_sum_all_mean = dist_inter_centers_sum_all_ / ( args.batch_size * nrof_centers_batch_) # print('Epoch: [%d][%d/%d]\tTime %.3f\tLoss %2.3f\tCrossEntropyLoss %2.3f\tRegLoss %2.3f\tdist_within_center %2.3f\tdist_inter_centers_sum_all_ (loss_center + beta -loss_inter_triplet centers) %2.3f (%2.3f + %2.3f - %2.3f) \tcenter_loss_factor %2.5f\tAcc %2.3f\tVal %2.3f\tFar %2.3f' % # (epoch, batch_number + 1, args.epoch_size, duration, totalloss, crossEntropyLoss, regloss, loss_center_, CenterTripletLoss, loss_center_, args.beta, loss_inter_centers_, args.center_loss_factor, acc, val, far)) # print('Epoch: [%d][%d/%d]\tTime %.3f\tLoss %2.3f\tCrossEntropyLoss %2.3f\tRegLoss %2.3f\tCenterInterLoss (loss_center/loss_inter_centers) * center_loss_factor %2.3f (%2.3f/%2.3f) * %2.3f\tAcc %2.3f\tVal %2.3f\tFar %2.3f' % # (epoch, batch_number+1, args.epoch_size, duration, err, err-np.sum(reg_loss), np.sum(reg_loss)-center_inter_loss * args.center_loss_factor,center_inter_loss, loss_center_, loss_inter_centers_, args.center_loss_factor, acc, val, far)) print( 'Epoch: [%d][%d/%d]\tTime %.3f\tTotal-loss %2.3f\tCrossEntropyLoss %2.3f\tRegLoss %2.3f\tClassTripletloss %2.3f\tdist_within_center_mean %2.3f\tdist_inter_centers_mean %2.3f\tnrof_centers_batch %2.3f\tcenter_loss_factor %2.5f\tAcc %2.3f\tVal %2.3f\tFar %2.3f\tbeta %2.3f \t Train_softmax_acc %2.3f \t lr: %f' % (epoch, batch_number + 1, args.epoch_size, duration, totalloss, crossEntropyLoss, Regloss, class_triplet_loss_, dist_within_center_mean, dist_inter_centers_sum_all_mean, nrof_centers_batch_, args.center_loss_factor, acc, val, far, args.beta, train_acc_, lr)) f.write( 'Epoch: [%d][%d/%d]\tTime %.3f\tTotal-loss %2.3f\tCrossEntropyLoss %2.3f\tRegLoss %2.3f\tClassTripletloss %2.3f\tdist_within_center_mean %2.3f\tdist_inter_centers_mean %2.3f\tnrof_centers_batch %2.3f\tcenter_loss_factor %2.5f\tAcc %2.3f\tVal %2.3f\tFar %2.3f\tbeta %2.3f \t Train_softmax_acc %2.3f \t lr %f' % (epoch, batch_number + 1, args.epoch_size, duration, totalloss, crossEntropyLoss, Regloss, class_triplet_loss_, dist_within_center_mean, dist_inter_centers_sum_all_mean, nrof_centers_batch_, args.center_loss_factor, acc, val, far, args.beta, train_acc_, lr)) batch_number += 1 train_time += duration # Add validation loss and accuracy to summary summary = tf.Summary() #pylint: disable=maybe-no-member summary.value.add(tag='time/total', simple_value=train_time) summary_writer.add_summary(summary, step) return step
import sys from facenet import get_learning_rate_from_file if __name__ == '__main__': lr = get_learning_rate_from_file(sys.argv[1], int(sys.argv[2])) print(lr)
def train(args, sess, dataset, epoch, image_paths_placeholder, labels_placeholder, labels_batch, batch_size_placeholder, learning_rate_placeholder, phase_train_placeholder, enqueue_op, input_queue, global_step, embeddings, loss, train_op, summary_op, summary_writer, learning_rate_schedule_file, embedding_size, anchor, positive, negative, triplet_loss): batch_number = 0 if args.learning_rate > 0.0: lr = args.learning_rate else: lr = facenet.get_learning_rate_from_file(learning_rate_schedule_file, epoch) k = 1 while k > 0: k -= 1 # while batch_number < args.epoch_size: #从数据集获取数据并随机打乱顺序 image_paths, num_per_class = sample_people(dataset, args.people_per_batch, args.images_per_person) print('Running forward pass on sampled images: ', end='') start_time = time.time() nrof_examples = args.people_per_batch * args.images_per_person # 数据总量 labels_array = np.reshape(np.arange(nrof_examples), (-1, 3)) image_paths_array = np.reshape( np.expand_dims(np.array(image_paths), 1), (-1, 3)) #将三元组组成的数据以(地址id,标签)格式入队 sess.run( enqueue_op, { image_paths_placeholder: image_paths_array, labels_placeholder: labels_array }) #enqueue_op.put(image_paths_array,labels_array) emb_array = np.zeros((nrof_examples, embedding_size)) # 根据batch大小划分的batch总量 nrof_batches = int(np.ceil(nrof_examples / args.batch_size)) #划分的batch数量 # 读入embedding,如果数据不足一个batch,以小量作为batch大小 # 读入通过sess传入的操作embeddings和labels for i in range(nrof_batches): batch_size = min(nrof_examples - i * args.batch_size, args.batch_size) phase_train = True emb, lab = sess.run( [embeddings, labels_batch], feed_dict={ batch_size_placeholder: batch_size, learning_rate_placeholder: lr, phase_train_placeholder: True }) emb_array[lab, :] = emb print('%.3f' % (time.time() - start_time)) # 根据embedding选择三元组 print('Selecting suitable triplets for training') triplets, nrof_random_negs, nrof_triplets = select_triplets( emb_array, num_per_class, image_paths, args.people_per_batch, args.alpha) selection_time = time.time() - start_time print( '(nrof_random_negs, nrof_triplets) = (%d, %d): time=%.3f seconds' % (nrof_random_negs, nrof_triplets, selection_time)) # 根据选择的三元组进行训练 nrof_batches = int(np.ceil(nrof_triplets * 3 / args.batch_size)) triplet_paths = list(itertools.chain(*triplets)) labels_array = np.reshape(np.arange(len(triplet_paths)), (-1, 3)) triplet_paths_array = np.reshape( np.expand_dims(np.array(triplet_paths), 1), (-1, 3)) # 将三元组添加到数据集 nrof_examples = len(triplet_paths) train_time = 0 i = 0 emb_array = np.zeros((nrof_examples, embedding_size)) loss_array = np.zeros((nrof_triplets, )) #summary = tf.Summary() step = 0 k = 1 while k > 0: k -= 1 #while i < nrof_batches: start_time = time.time() batch_size = min(nrof_examples - i * args.batch_size, args.batch_size) feed_dict = { batch_size_placeholder: batch_size, learning_rate_placeholder: lr, phase_train_placeholder: True } err, _, step, emb, lab = sess.run( [loss, train_op, global_step, embeddings, labels_batch], feed_dict=feed_dict) #进行下列四个操作函数 # gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=args.gpu_memory_fraction) # sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) # sess.run(train_op) # err=loss() # step=global_step # emb=embeddings # lab=labels_batch emb_array[lab, :] = emb loss_array[i] = err duration = time.time() - start_time print('Epoch: [%d][%d/%d]\tTime %.3f\tLoss %2.3f' % (epoch, batch_number + 1, args.epoch_size, duration, err)) batch_number += 1 i += 1 train_time += duration #summary.value.add(tag='loss', simple_value=err) # Add validation loss and accuracy to summary # pylint: disable=maybe-no-member #summary.value.add(tag='time/selection', simple_value=selection_time) #summary_writer.add_summary(summary, step) return step
def train(args, sess, epoch, image_list, label_list, index_dequeue_op, enqueue_op, image_paths_placeholder, labels_placeholder, learning_rate_placeholder, phase_train_placeholder, batch_size_placeholder, global_step, loss, accuracy, train_op, summary_op, summary_writer, regularization_losses, learning_rate_schedule_file, random_rotate, random_crop, random_flip, use_fixed_image_standardization, control_placeholder): batch_number = 0 if args.learning_rate > 0.0: lr = args.learning_rate else: lr = facenet.get_learning_rate_from_file(learning_rate_schedule_file, epoch) if lr <= 0: return False index_epoch = sess.run(index_dequeue_op) label_epoch = np.array(label_list)[index_epoch] image_epoch = np.array(image_list)[index_epoch] # Enqueue one epoch of image paths and labels labels_array = np.expand_dims(np.array(label_epoch), 1) image_paths_array = np.expand_dims(np.array(image_epoch), 1) control_value = facenet.RANDOM_ROTATE * random_rotate + facenet.RANDOM_CROP * random_crop + facenet.RANDOM_FLIP * random_flip + facenet.FIXED_STANDARDIZATION * use_fixed_image_standardization control_array = np.ones_like(labels_array) * control_value sess.run( enqueue_op, { image_paths_placeholder: image_paths_array, labels_placeholder: labels_array, control_placeholder: control_array }) print('training a epoch...') # Training loop train_time = 0 while batch_number < args.epoch_size: start_time = time.time() feed_dict = { learning_rate_placeholder: lr, phase_train_placeholder: True, batch_size_placeholder: args.batch_size } if batch_number % 100 == 0: err, accuracy_, _, step, reg_loss, summary_str = sess.run( [ loss, accuracy, train_op, global_step, regularization_losses, summary_op ], feed_dict=feed_dict) summary_writer.add_summary(summary_str, global_step=step) else: err, accuracy_, _, step, reg_loss = sess.run( [loss, accuracy, train_op, global_step, regularization_losses], feed_dict=feed_dict) duration = time.time() - start_time print( 'Epoch: [%d][%d/%d]\tTime %.3f\tLoss %2.3f\tRegLoss %2.3f\tAccuracy %2.3f' % (epoch, batch_number + 1, args.epoch_size, duration, err, np.sum(reg_loss), accuracy_)) batch_number += 1 train_time += duration # Add validation loss and accuracy to summary summary = tf.Summary() #pylint: disable=maybe-no-member summary.value.add(tag='time/total', simple_value=train_time) summary_writer.add_summary(summary, step) return True
def train(args, sess, dataset, epoch, image_paths_placeholder, labels_placeholder, labels_batch, batch_size_placeholder, learning_rate_placeholder, phase_train_placeholder, enqueue_op, input_queue, global_step, embeddings, loss, train_op, summary_op, summary_writer, learning_rate_schedule_file, embedding_size, anchor, positive, negative, triplet_loss): batch_number = 0 if args.learning_rate>0.0: lr = args.learning_rate else: lr = facenet.get_learning_rate_from_file(learning_rate_schedule_file, epoch) while batch_number < args.epoch_size: # Sample people randomly from the dataset image_paths, num_per_class = sample_people(dataset, args.people_per_batch, args.images_per_person) print('Running forward pass on sampled images: ', end='') start_time = time.time() nrof_examples = args.people_per_batch * args.images_per_person labels_array = np.reshape(np.arange(nrof_examples),(-1,3)) image_paths_array = np.reshape(np.expand_dims(np.array(image_paths),1), (-1,3)) sess.run(enqueue_op, {image_paths_placeholder: image_paths_array, labels_placeholder: labels_array}) emb_array = np.zeros((nrof_examples, embedding_size)) nrof_batches = int(np.ceil(nrof_examples / args.batch_size)) for i in xrange(nrof_batches): batch_size = min(nrof_examples-i*args.batch_size, args.batch_size) emb, lab = sess.run([embeddings, labels_batch], feed_dict={batch_size_placeholder: batch_size, learning_rate_placeholder: 0.0, phase_train_placeholder: False}) emb_array[lab,:] = emb print('%.3f' % (time.time()-start_time)) # Select triplets based on the embeddings print('Selecting suitable triplets for training') triplets, nrof_random_negs, nrof_triplets = select_triplets(emb_array, num_per_class, image_paths, args.people_per_batch, args.alpha) selection_time = time.time() - start_time print('(nrof_random_negs, nrof_triplets) = (%d, %d): time=%.3f seconds' % (nrof_random_negs, nrof_triplets, selection_time)) # Perform training on the selected triplets nrof_batches = int(np.ceil(nrof_triplets*3/args.batch_size)) triplet_paths = list(itertools.chain(*triplets)) #triplet_paths = triplet_paths[0:nrof_batches*args.batch_size] print('Nrof paths: %d' % len(triplet_paths)) labels_array = np.reshape(np.arange(len(triplet_paths)),(-1,3)) triplet_paths_array = np.reshape(np.expand_dims(np.array(triplet_paths),1), (-1,3)) sess.run(enqueue_op, {image_paths_placeholder: triplet_paths_array, labels_placeholder: labels_array}) nrof_examples = len(triplet_paths) train_time = 0 i = 0 emb_array = np.zeros((nrof_examples, embedding_size)) loss_array = np.zeros((nrof_triplets,)) while i < nrof_batches: start_time = time.time() batch_size = min(nrof_examples-i*args.batch_size, args.batch_size) feed_dict = {batch_size_placeholder: batch_size, learning_rate_placeholder: lr, phase_train_placeholder: True} err, _, step, emb, lab = sess.run([loss, train_op, global_step, embeddings, labels_batch], feed_dict=feed_dict) emb_array[lab,:] = emb loss_array[i] = err duration = time.time() - start_time print('Epoch: [%d][%d/%d]\tTime %.3f\tLoss %2.3f' % (epoch, batch_number+1, args.epoch_size, duration, err)) batch_number += 1 i += 1 train_time += duration # Add validation loss and accuracy to summary summary = tf.Summary() #pylint: disable=maybe-no-member summary.value.add(tag='time/selection', simple_value=selection_time) summary_writer.add_summary(summary, step) return step
def train(args, sess, dataset, epoch, image_paths_placeholder, labels_placeholder, labels_batch, batch_size_placeholder, learning_rate_placeholder, phase_train_placeholder, enqueue_op, input_queue, global_step, embeddings, loss, train_op, summary_op, summary_writer, learning_rate_schedule_file, embedding_size, anchor, positive, negative, triplet_loss): batch_number = 0 # 学习率大于 0 if args['learning_rate'] > 0.0: lr = args['learning_rate'] else: lr = facenet.get_learning_rate_from_file(learning_rate_schedule_file, epoch) # 运行一个批处理 while batch_number < args['epoch_size']: # 随机选取数据 # 1800 个 image_paths, num_per_class = sample_people(dataset, args['people_per_batch'], args['images_per_person']) print('进行前向传播:', end='') start_time = time.time() examples = args['people_per_batch'] * args['images_per_person'] # 将输入的1800个图像变成了600*3的二维数据, # reshape(a,(-1,3))表示只给定列数为3, 行数自行算出 labels_array = np.reshape(np.arange(examples), (-1, 3)) image_paths_array = np.reshape( np.expand_dims(np.array(image_paths), 1), (-1, 3)) # 开辟一个新的线程用于在内存里读取数据 sess.run( enqueue_op, { image_paths_placeholder: image_paths_array, labels_placeholder: labels_array }) emb_array = np.zeros((examples, embedding_size)) # nrof_batches,1800/90=20 nrof_batches = int(np.ceil(examples / args['batch_size'])) # 批处理取得人脸特征,默认为20个批 for i in range(nrof_batches): batch_size = min(examples - i * args['batch_size'], args['batch_size']) emb, lab = sess.run( [embeddings, labels_batch], feed_dict={ batch_size_placeholder: batch_size, learning_rate_placeholder: lr, phase_train_placeholder: True }) emb_array[lab, :] = emb print('%.3f' % (time.time() - start_time)) print('选择合适的三元组') triplets, nrof_random_negs, nrof_triplets = select_triplets( emb_array, num_per_class, image_paths, args['people_per_batch'], args['alpha']) selection_time = time.time() - start_time print('(random_negs, triplets) = (%d, %d): time=%.3f seconds' % (nrof_random_negs, nrof_triplets, selection_time)) # 使用选定的三元组训练 nrof_batches = int(np.ceil(nrof_triplets * 3 / args['batch_size'])) triplet_paths = list(itertools.chain(*triplets)) labels_array = np.reshape(np.arange(len(triplet_paths)), (-1, 3)) triplet_paths_array = np.reshape( np.expand_dims(np.array(triplet_paths), 1), (-1, 3)) # 读取数据的操作 sess.run( enqueue_op, { image_paths_placeholder: triplet_paths_array, labels_placeholder: labels_array }) examples = len(triplet_paths) train_time = 0 i = 0 emb_array = np.zeros((examples, embedding_size)) loss_array = np.zeros((nrof_triplets, )) # 根据求出的特征计算triplet损失函数并进行优化 summary = tf.Summary() while i < nrof_batches: start_time = time.time() batch_size = min(examples - i * args['batch_size'], args['batch_size']) feed_dict = { batch_size_placeholder: batch_size, learning_rate_placeholder: lr, phase_train_placeholder: True } # sess run 有5个输入,fetches,先运行loss # 前向计算的损失,train_op是根据损失来计算梯度,来对参数进行优化 err, _, step, emb, lab = sess.run( [loss, train_op, global_step, embeddings, labels_batch], feed_dict=feed_dict) emb_array[lab, :] = emb loss_array[i] = err duration = time.time() - start_time print('Epoch: [%d][%d/%d]\tTime %.3f\tLoss %2.3f' % (epoch, batch_number + 1, args['epoch_size'], duration, err)) batch_number += 1 i += 1 train_time += duration summary.value.add(tag='loss', simple_value=err) # 将验证损失和准确性添加到summary summary.value.add(tag='time/selection', simple_value=selection_time) summary_writer.add_summary(summary, step) return step
def train_multi_gpus(args, sess, dataset, epoch, image_paths_placeholder, labels_placeholder, labels_batch, batch_size_placeholder, learning_rate_placeholder, phase_train_placeholder, enqueue_op, input_queue, global_step, embeddings, anchors, positives, negatives, train_op, summary_op, summary_writer, learning_rate_schedule_file, embedding_size): batch_number = 0 if args.learning_rate > 0.0: lr = args.learning_rate else: lr = facenet.get_learning_rate_from_file(learning_rate_schedule_file, epoch) while batch_number < args.epoch_size: # Sample people randomly from the dataset image_paths, num_per_class = sample_people(dataset, args.people_per_batch, args.images_per_person) print('Running forward pass on sampled images: ', end='') start_time = time.time() nrof_examples = args.people_per_batch * args.images_per_person labels_array = np.reshape(np.arange(nrof_examples), (-1, 3)) image_paths_array = np.reshape( np.expand_dims(np.array(image_paths), 1), (-1, 3)) sess.run( enqueue_op, { image_paths_placeholder: image_paths_array, labels_placeholder: labels_array }) emb_array = np.zeros((nrof_examples, embedding_size)) nrof_batches = int(np.ceil(nrof_examples / args.batch_size)) for i in range(nrof_batches): batch_size = min(nrof_examples - i * args.batch_size, args.batch_size) emb, lab = sess.run( [embeddings, labels_batch], feed_dict={ batch_size_placeholder: batch_size, learning_rate_placeholder: lr, phase_train_placeholder: True }) emb_array[lab, :] = emb print('加载数据完毕,用时 %.3f' % (time.time() - start_time)) # Select triplets based on the embeddings print('Selecting suitable triplets for training...') triplets, nrof_random_negs, nrof_triplets = select_triplets( emb_array, num_per_class, image_paths, args.people_per_batch, args.alpha) selection_time = time.time() - start_time print( '选择 三元组 完毕 --- (nrof_random_negs, nrif_triplets) = (%d, %d): time=%.3f seconds' % (nrof_random_negs, nrof_triplets, selection_time)) # 在选定的 triplets 上执行 训练 nrof_batches = int(np.ceil(nrof_triplets * 3 / args.batch_size)) print('选择出来的 triplets 形成的 batch 有多少个:', nrof_batches) triplet_paths = list(itertools.chain(*triplets)) # print('triplet_paths:', triplet_paths) labels_array = np.reshape(np.arange(len(triplet_paths)), (-1, 3)) triplet_paths_array = np.reshape( np.expand_dims(np.array(triplet_paths), 1), (-1, 3)) # 把 选择出来的 triplets 添加到 enqueue 中去 sess.run( enqueue_op, { image_paths_placeholder: triplet_paths_array, labels_placeholder: labels_array }) nrof_examples = len(triplet_paths) print('选择出的 triplets 的个数为:', nrof_examples) train_time = 0 i = 0 emb_array = np.zeros((nrof_examples, embedding_size)) loss_array = np.zeros((nrof_triplets, )) summary = tf.Summary() step = 0 while i < nrof_batches: start_time = time.time() # 添加 计算 loss 的代码,加在此处,是因为每个 epoch 中的每个 batch 都要计算一下 loss # First, calculate the total_loss to run the following code # 在这部分修改为 multi-gpu 训练 # 即 将每一个 batch 均分,然后在多个 gpu 上来计算对应的 triplet_loss ,之后汇总得到和,求取平均,得到一个 batch_size 的 loss tower_losses = [] tower_triplets = [] tower_reg = [] with tf.variable_scope(tf.get_variable_scope()): for i in range(args.num_gpus): with tf.device('/gpu:' + str(i + 2)): with tf.name_scope('tower_' + str(i)) as scope: print('###' * 10, i) print('---' * 10, anchors[i]) print('---' * 10, positives[i]) print('---' * 10, negatives[i]) print('###' * 10, i) triplet_loss_split = facenet.triplet_loss( anchors[i], positives[i], negatives[i], args.alpha) regularization_loss = tf.get_collection( tf.GraphKeys.REGULARIZATION_LOSSES) # 添加本行代码,进行同名变量的重用 tf.get_variable_scope().reuse_variables() tower_triplets.append(triplet_loss_split) loss = triplet_loss_split + tf.add_n( regularization_loss) tower_losses.append(loss) tower_reg.append(regularization_loss) # 计算 multi gpu 运行完成得到的 loss total_loss = tf.reduce_mean(tower_losses) total_reg = tf.reduce_mean(tower_reg) losses = {} losses['total_loss'] = total_loss losses['total_reg'] = total_reg loss = total_loss batch_size = min(nrof_examples - i * args.batch_size, args.batch_size) feed_dict = { batch_size_placeholder: batch_size, learning_rate_placeholder: lr, phase_train_placeholder: True } # --- print start --- print('**************batch_size', batch_size) print('**************lr', lr) print('**************loss', loss, loss.get_shape()) print('**************train_op', train_op) print('**************global_step', global_step) print('**************embeddings', embeddings, embeddings.get_shape()) print('**************labels_batch', labels_batch, labels_batch.get_shape()) # --- print end ----- err, _, step, emb, lab = sess.run( [loss, train_op, global_step, embeddings, labels_batch], feed_dict=feed_dict) emb_array[lab, :] = emb loss_array[i] = err duration = time.time() - start_time print('Epoch: [%d][%d/%d]\tTime %.3f\tLoss %2.3f' % (epoch, batch_number + 1, args.epoch_size, duration, err)) batch_number += 1 i += 1 train_time += duration summary.value.add(tag='loss', simple_value=err) # Add validation loss and accuracy to summary #pylint: disable=maybe-no-member summary.value.add(tag='time/selection', simple_value=selection_time) summary_writer.add_summary(summary, step) return step
def train(args, sess, dataset, epoch, image_paths_placeholder, labels_placeholder, labels_batch, batch_size_placeholder, learning_rate_placeholder, phase_train_placeholder, enqueue_op, global_step, embeddings, loss, train_op, summary_writer, regularization_losses, learning_rate_schedule_file, embedding_size, nrof_classes, summary_op): batch_number = 0 if args.learning_rate > 0.0: lr = args.learning_rate else: lr = facenet.get_learning_rate_from_file(learning_rate_schedule_file, epoch) triplet_batch_number = 0 ## mzh edit 31012017 while batch_number < args.epoch_size: #epoch_size is not the step size of each epoch but the number of the batch (of the triplet) to be processed triplet_batch_number += 1 # Sample people randomly from the dataset image_paths, num_per_class = sample_people(dataset, args.people_per_batch, args.images_per_person) print('Running forward pass on sampled images: ', end='') start_time = time.time() nrof_examples = args.people_per_batch * args.images_per_person labels_array = np.reshape(np.arange(nrof_examples), (-1, 3)) image_paths_array = np.reshape( np.expand_dims(np.array(image_paths), 1), (-1, 3)) sess.run( enqueue_op, { image_paths_placeholder: image_paths_array, labels_placeholder: labels_array }) emb_array = np.zeros((nrof_examples, embedding_size)) nrof_batches = int(np.ceil(nrof_examples / args.batch_size)) ## Construct the em,lab, the data will be feed later for i in xrange(nrof_batches): batch_size = min(nrof_examples - i * args.batch_size, args.batch_size) emb, lab = sess.run( [embeddings, labels_batch], feed_dict={ batch_size_placeholder: batch_size, learning_rate_placeholder: lr, phase_train_placeholder: True }) emb_array[lab, :] = emb print('%.3f' % (time.time() - start_time)) # Select triplets based on the embeddings print('Selecting suitable triplets for training') triplets, nrof_random_negs, nrof_triplets = select_triplets( emb_array, num_per_class, image_paths, args.people_per_batch, args.alpha) selection_time = time.time() - start_time print( '(nrof_random_negs, nrof_triplets) = (%d, %d): time=%.3f seconds' % (nrof_random_negs, nrof_triplets, selection_time)) # Perform training on the selected triplets nrof_batches = int(np.ceil(nrof_triplets * 3 / args.batch_size)) triplet_paths = list(itertools.chain(*triplets)) ## The labels_arrays for tripletloss is the number of the elements in the triplet_paths mzh 31012017 #labels_array = np.reshape(np.arange(len(triplet_paths)),(-1,3)) ## The labels_arrays for center loss is the number of the class (i.e. persons) corresponding to each elements in the triplet_paths mzh 31012017 labels_triplet = facenet.get_label_triplet(triplet_paths) assert (max(labels_triplet) <= nrof_classes) labels_array = np.reshape(labels_triplet, (-1, 3)) triplet_paths_array = np.reshape( np.expand_dims(np.array(triplet_paths), 1), (-1, 3)) sess.run( enqueue_op, { image_paths_placeholder: triplet_paths_array, labels_placeholder: labels_array }) nrof_examples = len(triplet_paths) train_time = 0 i = 0 emb_array = np.zeros((nrof_examples, embedding_size)) loss_array = np.zeros((nrof_triplets, )) while i < nrof_batches: start_time = time.time() batch_size = min(nrof_examples - i * args.batch_size, args.batch_size) feed_dict = { batch_size_placeholder: batch_size, learning_rate_placeholder: lr, phase_train_placeholder: True } ## The joint triplet loss + center loss mzh 31012017 err, _, step, emb, lab, reg_loss = sess.run([ loss, train_op, global_step, embeddings, labels_batch, regularization_losses ], feed_dict=feed_dict) ## Only the triplet loss #err, _, step, emb, lab = sess.run([loss, train_op, global_step, embeddings, labels_batch], feed_dict=feed_dict) #emb_array[lab,:] = emb # if (batch_number % 100 == 0): # summary_str, step = sess.run([summary_op, global_step], feed_dict=feed_dict) # summary_writer.add_summary(summary_str, global_step=step) total_loss = err center_loss = np.sum(reg_loss) triple_loss = err - center_loss loss_array[i] = err duration = time.time() - start_time # print('Epoch: [%d][%d/%d]\tTime %.3f\tLoss %2.3f' % # (epoch, batch_number+1, args.epoch_size, duration, err)) # print('Epoch: [%d][%d/%d]\tTime %.3f\tLoss %2.3f\tRegLoss %2.3f' % # (epoch, batch_number+1, args.epoch_size, duration, err, np.sum(reg_loss))) print( 'Epoch: [%d][%d/%d/%d][%d/%d]\tTime %.3f\tTotalloss %2.3f\tTripletloss %2.3f\tCenterloss %2.3f' % (epoch, i, nrof_batches, triplet_batch_number, batch_number + 1, args.epoch_size, duration, total_loss, triple_loss, center_loss)) ## mzh edit, 31012017 batch_number += 1 i += 1 train_time += duration # Add validation loss and accuracy to summary summary = tf.Summary() #pylint: disable=maybe-no-member summary.value.add(tag='time/selection', simple_value=selection_time) ## edit mzh # summary.value.add(tag='Loss/Total_loss', simple_value=total_loss) # summary.value.add(tag='Loss/Triplet_loss', simple_value=triple_loss) # summary.value.add(tag='Loss/Center_loss', simple_value=center_loss) # summary.value.add(tag='Learning_rate', simple_value=lr) summary_writer.add_summary(summary, step) return step
def train(args, sess, dataset, epoch, image_paths_placeholder, labels_placeholder, labels_batch, batch_size_placeholder, learning_rate_placeholder, phase_train_placeholder, enqueue_op, input_queue, global_step, embeddings, loss, train_op, summary_op, summary_writer, learning_rate_schedule_file, embedding_size, anchor, positive, negative, triplet_loss): batch_number = 0 if args.learning_rate > 0.0: lr = args.learning_rate else: lr = facenet.get_learning_rate_from_file(learning_rate_schedule_file, epoch) while batch_number < args.epoch_size: # Sample people randomly from the dataset image_paths, num_per_class = sample_people(dataset, args.people_per_batch, args.images_per_person) print('Running forward pass on sampled images: ', end='') start_time = time.time() nrof_examples = args.people_per_batch * args.images_per_person labels_array = np.reshape(np.arange(nrof_examples), (-1, 3)) image_paths_array = np.reshape( np.expand_dims(np.array(image_paths), 1), (-1, 3)) sess.run( enqueue_op, { image_paths_placeholder: image_paths_array, labels_placeholder: labels_array }) emb_array = np.zeros((nrof_examples, embedding_size)) nrof_batches = int(np.ceil(nrof_examples / args.batch_size)) for i in range(nrof_batches): batch_size = min(nrof_examples - i * args.batch_size, args.batch_size) emb, lab = sess.run( [embeddings, labels_batch], feed_dict={ batch_size_placeholder: batch_size, learning_rate_placeholder: lr, phase_train_placeholder: True }) emb_array[lab, :] = emb print('%.3f' % (time.time() - start_time)) # Select triplets based on the embeddings print('Selecting suitable triplets for training') triplets, nrof_random_negs, nrof_triplets = select_triplets( emb_array, num_per_class, image_paths, args.people_per_batch, args.alpha) selection_time = time.time() - start_time print( '(nrof_random_negs, nrof_triplets) = (%d, %d): time=%.3f seconds' % (nrof_random_negs, nrof_triplets, selection_time)) # Perform training on the selected triplets nrof_batches = int(np.ceil(nrof_triplets * 3 / args.batch_size)) triplet_paths = list(itertools.chain(*triplets)) labels_array = np.reshape(np.arange(len(triplet_paths)), (-1, 3)) triplet_paths_array = np.reshape( np.expand_dims(np.array(triplet_paths), 1), (-1, 3)) sess.run( enqueue_op, { image_paths_placeholder: triplet_paths_array, labels_placeholder: labels_array }) nrof_examples = len(triplet_paths) train_time = 0 i = 0 emb_array = np.zeros((nrof_examples, embedding_size)) loss_array = np.zeros((nrof_triplets, )) summary = tf.Summary() step = 0 while i < nrof_batches: start_time = time.time() batch_size = min(nrof_examples - i * args.batch_size, args.batch_size) feed_dict = { batch_size_placeholder: batch_size, learning_rate_placeholder: lr, phase_train_placeholder: True } err, _, step, emb, lab = sess.run( [loss, train_op, global_step, embeddings, labels_batch], feed_dict=feed_dict) emb_array[lab, :] = emb loss_array[i] = err duration = time.time() - start_time print('Epoch: [%d][%d/%d]\tTime %.3f\tLoss %2.3f' % (epoch, batch_number + 1, args.epoch_size, duration, err)) batch_number += 1 i += 1 train_time += duration summary.value.add(tag='loss', simple_value=err) # Add validation loss and accuracy to summary #pylint: disable=maybe-no-member summary.value.add(tag='time/selection', simple_value=selection_time) summary_writer.add_summary(summary, step) return step
def train(args, sess, dataset, epoch, images_placeholder, learning_rate_placeholder, global_step, embeddings, loss, train_op, summary_op, summary_writer): batch_number = 0 if args.learning_rate > 0.0: lr = args.learning_rate else: lr = facenet.get_learning_rate_from_file( '../data/learning_rate_schedule.txt', epoch) while batch_number < args.epoch_size: print('Loading training data') # Sample people and load new data start_time = time.time() image_paths, num_per_class = facenet.sample_people( dataset, args.people_per_batch, args.images_per_person) image_data = facenet.load_data(image_paths, args.random_crop, args.random_flip, args.image_size) load_time = time.time() - start_time print('Loaded %d images in %.2f seconds' % (image_data.shape[0], load_time)) print('Selecting suitable triplets for training') start_time = time.time() emb_list = [] # Run a forward pass for the sampled images nrof_examples_per_epoch = args.people_per_batch * args.images_per_person nrof_batches_per_epoch = int( np.floor(nrof_examples_per_epoch / args.batch_size)) for i in xrange(nrof_batches_per_epoch): batch = facenet.get_batch(image_data, args.batch_size, i) feed_dict = { images_placeholder: batch, learning_rate_placeholder: lr } emb_list += sess.run([embeddings], feed_dict=feed_dict) emb_array = np.vstack( emb_list ) # Stack the embeddings to a nrof_examples_per_epoch x 128 matrix # Select triplets based on the embeddings triplets, nrof_random_negs, nrof_triplets = facenet.select_triplets( emb_array, num_per_class, image_data, args.people_per_batch, args.alpha) selection_time = time.time() - start_time print( '(nrof_random_negs, nrof_triplets) = (%d, %d): time=%.3f seconds' % (nrof_random_negs, nrof_triplets, selection_time)) # Perform training on the selected triplets train_time = 0 i = 0 while i * args.batch_size < nrof_triplets * 3 and batch_number < args.epoch_size: start_time = time.time() batch = facenet.get_triplet_batch(triplets, i, args.batch_size) feed_dict = { images_placeholder: batch, learning_rate_placeholder: lr } err, _, step = sess.run([loss, train_op, global_step], feed_dict=feed_dict) if (batch_number % 100 == 0): summary_str, step = sess.run([summary_op, global_step], feed_dict=feed_dict) summary_writer.add_summary(summary_str, global_step=step) duration = time.time() - start_time print('Epoch: [%d][%d/%d]\tTime %.3f\tLoss %2.3f' % (epoch, batch_number + 1, args.epoch_size, duration, err)) batch_number += 1 i += 1 train_time += duration # Add validation loss and accuracy to summary summary = tf.Summary() #pylint: disable=maybe-no-member summary.value.add(tag='time/load', simple_value=load_time) summary.value.add(tag='time/selection', simple_value=selection_time) summary.value.add(tag='time/train', simple_value=train_time) summary.value.add(tag='time/total', simple_value=load_time + selection_time + train_time) summary_writer.add_summary(summary, step) return step
def train(args, sess, dataset, epoch, image_placeholder, code_placeholder, batch_size_placeholder, learning_rate_placeholder, phase_train_placeholder, global_step, embeddings, loss, train_op, summary_op, summary_writer, learning_rate_schedule_file, embedding_size, anchor, positive, negative, triplet_loss, triplets_list, code_list,model_dir,Accuracy): batch_number = 0 images_data = h5py.File('../face_hdf5_150by150/train_classification.h5') images = images_data['data'] if args.learning_rate>0.0: lr = args.learning_rate else: lr = facenet.get_learning_rate_from_file(learning_rate_schedule_file, epoch) start_time = time.time() # sess.run(enqueue_op, {image_paths_placeholder: image_paths_array, labels_placeholder: labels_array}) nrof_examples = 0 class_num = len(triplets_list) # for i in range(class_num): nrof_examples = max(len(triplets_list[0]), nrof_examples) nrof_batches = int(np.ceil(nrof_examples / (args.batch_size/3/class_num))) print('load mean image done!') train_time = 0 batch_number = 0 batch_size = args.batch_size # emb_array = np.zeros((nrof_examples, embedding_size)) # loss_array = np.zeros((nrof_triplets,)) while batch_number < nrof_batches: start_time = time.time() class_num = len(triplets_list) image_array = np.zeros((batch_size,150,150,3),np.float32) code_array = np.zeros((0,class_num,1,1), np.float32) length = [] for t in range(class_num): triplets = triplets_list[t] length.append(len(triplets)) start_index = t*int(batch_size/class_num) image_array = facenet.get_triplet_image_batch1(images, triplets, batch_size/class_num, batch_number, image_array, start_index, args) code = code_list[t] code_array_tem = code[0:int(batch_size/class_num)] code_array = np.vstack((code_array, code_array_tem)) time1 = time.time() - start_time feed_dict = {batch_size_placeholder: batch_size, learning_rate_placeholder: lr, phase_train_placeholder: True, image_placeholder: image_array, code_placeholder: code_array} err, _, step, emb = sess.run([loss, train_op, global_step, embeddings], feed_dict=feed_dict) # emb_array[lab,:] = emb # loss_array[i] = err emb_feature = emb nrof_pair = emb_feature.shape[0]/3 correct_num = 0 for pair_index in range(0, int(nrof_pair/class_num)): anchor_index = pair_index*3 pos_index = anchor_index + 1 neg_index = anchor_index + 2 an_feature = emb_feature[anchor_index,:] pos_feature = emb_feature[pos_index, :] neg_feature = emb_feature[neg_index, :] if np.sum(np.square(an_feature - pos_feature),0) < np.sum(np.square(an_feature - neg_feature),0): correct_num = correct_num + 1 p_dist = np.sum(np.square(an_feature - pos_feature), 0) n_dist = np.sum(np.square(an_feature - neg_feature),0) print('triplet number:', length) print('pos_distance and neg_distance', p_dist, n_dist, 'model_dir:',model_dir,'last accuracy:',Accuracy[-1]) accuracy = correct_num/(nrof_pair/class_num) duration = time.time() - start_time print('Epoch: [%d][%d/%d]\tTime %.3f\tread_time %.3f\tAccuracy %2.3f\tlr %2.3f\tLoss %2.3f' % (epoch, batch_number+1, nrof_batches, duration, time1, accuracy, lr, err)) batch_number += 1 train_time += duration # Add validation loss and accuracy to summary summary = tf.Summary() #pylint: disable=maybe-no-member #summary.value.add(tag='time/selection', simple_value=train_time) summary_writer.add_summary(summary, step) return step
def train(self, image_batch, args, sess, data_source, unsupervised, supervised_dataset, unsupervised_dataset, epoch, image_paths_placeholder, labels_placeholder, data_augmentations_placeholder, source_image_paths_placeholder, target_image_paths_placeholder, labels_batch, batch_size_placeholder, learning_rate_placeholder, phase_train_placeholder, enqueue_op, domain_enqueue_op, global_step, embeddings, loss, optimizer, gradients, gradient_placeholder, apply_gradient_op, summary_op, summary_writer, learning_rate_schedule_file, embedding_size, triplet_loss, domain_adaptation_loss, log_dir): if args.learning_rate > 0.0: lr = args.learning_rate else: lr = facenet.get_learning_rate_from_file( learning_rate_schedule_file, epoch) batch_number = 0 while batch_number < args.epoch_size: # Sample people randomly from the dataset image_paths, num_per_class = self.sample_people( data_source, supervised_dataset, args.people_per_batch, args.images_per_person) logger.debug('Running forward pass on sampled images: ') start_time = time.time() nrof_examples = args.people_per_batch * args.images_per_person labels_array = np.reshape(np.arange(nrof_examples), (-1, 3)) image_paths_array = np.reshape( np.expand_dims(np.array(image_paths), 1), (-1, 3)) data_augmentations_array = self.get_data_augmentations_array( labels_array) sess.run(enqueue_op, feed_dict={ image_paths_placeholder: image_paths_array, labels_placeholder: labels_array, data_augmentations_placeholder: data_augmentations_array }) emb_array = np.zeros((nrof_examples, embedding_size)) nrof_batches = int(np.ceil(nrof_examples / args.batch_size)) for i in range(nrof_batches): batch_size = min(nrof_examples - i * args.batch_size, args.batch_size) emb, lab = sess.run( [embeddings, labels_batch], feed_dict={ batch_size_placeholder: batch_size, learning_rate_placeholder: lr, phase_train_placeholder: True }) # logger.debug('labels_batch: %s' % (lab)) emb_array[lab, :] = emb logger.debug('train time: %.3f' % (time.time() - start_time)) # Select triplets based on the embeddings logger.debug('Selecting suitable triplets for training') # triplets, nrof_random_negs, nrof_triplets = self.select_triplets(emb_array, num_per_class, # image_paths, args.people_per_batch, args.alpha) triplets, nrof_random_negs, nrof_triplets = self.select_triplets_ceiling( emb_array, num_per_class, image_paths, args.people_per_batch, args.alpha, self.nrof_triplets_gap) selection_time = time.time() - start_time logger.debug( '(nrof_random_negs, nrof_triplets) = (%d, %d): time=%.3f seconds' % (nrof_random_negs, nrof_triplets, selection_time)) self.triplets = self.triplets + triplets self.nrof_triplets_gap = self.nrof_triplets_gap - nrof_triplets # logger.debug('triplets: %s' % (self.triplets)) logger.debug('nrof_triplets_gap: %s' % (self.nrof_triplets_gap)) if self.nrof_triplets_gap > 0: logger.debug('not enough triplets...') continue # Perform training on the selected triplets # nrof_batches = int(np.ceil(nrof_triplets * 3 / args.batch_size)) nrof_batches = int( np.ceil(self.max_triplet_per_select * 3 / args.batch_size)) triplet_paths = list(itertools.chain(*self.triplets)) labels_array = np.reshape(np.arange(len(triplet_paths)), (-1, 3)) triplet_paths_array = np.reshape( np.expand_dims(np.array(triplet_paths), 1), (-1, 3)) data_augmentations_array = self.get_data_augmentations_array( labels_array) sess.run(enqueue_op, feed_dict={ image_paths_placeholder: triplet_paths_array, labels_placeholder: labels_array, data_augmentations_placeholder: data_augmentations_array }) if args.data_source == 'MULTIPLE' and args.unsupervised != 'NONE': source_image_paths, target_image_paths = self.sample_unsupervised_dataset( unsupervised_dataset, nrof_batches * args.batch_size // 2) logger.debug("before domain_enqueue_op") source_image_paths_array = np.expand_dims( np.array(source_image_paths), 1) target_image_paths_array = np.expand_dims( np.array(target_image_paths), 1) sess.run( domain_enqueue_op, { source_image_paths_placeholder: source_image_paths_array, target_image_paths_placeholder: target_image_paths_array }) logger.debug("after domain_enqueue_op") nrof_examples = len(triplet_paths) train_time = 0 i = 0 # emb_array = np.zeros((nrof_examples, embedding_size)) # loss_array = np.zeros((nrof_triplets,)) summary = tf.Summary() step = 0 while i < nrof_batches: start_time = time.time() batch_size = min(nrof_examples - i * args.batch_size, args.batch_size) feed_dict = { batch_size_placeholder: batch_size, # round_placeholder: self._round, learning_rate_placeholder: lr, phase_train_placeholder: True } images, err, grads, step, emb, lab, triplet_loss_, domain_adaptation_loss_ = sess.run( [ image_batch, loss, gradients, global_step, embeddings, labels_batch, triplet_loss, domain_adaptation_loss ], feed_dict=feed_dict) # grads, _ = zip(*g_and_v) self.gradient_buffer = [ sum(x) for x in zip(self.gradient_buffer, grads) ] self.nrof_gradients += 1 # emb_array[lab, :] = emb # loss_array[i] = err duration = time.time() - start_time # if args.random_flip: # self.save_images(images, batch_number) logger.debug( 'Epoch [%d][%d/%d]\tRound [%d]\tgrad: [%d]\tTime %.3f\tLoss %f\ttriplet_Loss %f\tdomain_adaptation_loss %f' % (epoch, batch_number + 1, args.epoch_size, self._round, self.nrof_gradients, duration, err, triplet_loss_, domain_adaptation_loss_)) batch_number += 1 i += 1 train_time += duration summary.value.add(tag='loss', simple_value=err) # Add validation loss and accuracy to summary # pylint: disable=maybe-no-member summary.value.add(tag='time/selection', simple_value=selection_time) summary_writer.add_summary(summary, step) self._round += 1 if self.nrof_gradients > 0: # if self.nrof_gradients > 0 and self._round % 3 == 0: logger.info("apply_gradient...") self.gradient_buffer = map(lambda x: x / self.nrof_gradients, self.gradient_buffer) feed_dict = {learning_rate_placeholder: lr} feed_dict_grad = { grad_placeholder: grad for grad_placeholder, grad in zip(gradient_placeholder, self.gradient_buffer) } feed_dict.update(feed_dict_grad) if args.cluster: _, lr_ = sess.run( [apply_gradient_op, optimizer._optimizer._lr], feed_dict=feed_dict) else: _, lr_ = sess.run([apply_gradient_op, optimizer._lr], feed_dict=feed_dict) with open(os.path.join(log_dir, 'learing_rate.txt'), 'at') as f: f.write('%d\t%.5f\n' % (epoch, lr_)) self.gradient_buffer = [0.0] * self.nrof_trainable_vars self.nrof_gradients = 0 self.nrof_triplets_gap = self.max_triplet_per_select self.triplets = [] return step
def train(args, sess, dataset, epoch, images_placeholder, learning_rate_placeholder, global_step, embeddings, loss, train_op, summary_op, summary_writer): batch_number = 0 if args.learning_rate>0.0: lr = args.learning_rate else: lr = facenet.get_learning_rate_from_file('../data/learning_rate_schedule.txt', epoch) while batch_number < args.epoch_size: print('Loading training data') # Sample people and load new data start_time = time.time() image_paths, num_per_class = facenet.sample_people(dataset, args.people_per_batch, args.images_per_person) image_data = facenet.load_data(image_paths, args.random_crop, args.random_flip, args.image_size) load_time = time.time() - start_time print('Loaded %d images in %.2f seconds' % (image_data.shape[0], load_time)) print('Selecting suitable triplets for training') start_time = time.time() emb_list = [] # Run a forward pass for the sampled images nrof_examples_per_epoch = args.people_per_batch * args.images_per_person nrof_batches_per_epoch = int(np.floor(nrof_examples_per_epoch / args.batch_size)) for i in xrange(nrof_batches_per_epoch): batch = facenet.get_batch(image_data, args.batch_size, i) feed_dict = {images_placeholder: batch, learning_rate_placeholder: lr} emb_list += sess.run([embeddings], feed_dict=feed_dict) emb_array = np.vstack(emb_list) # Stack the embeddings to a nrof_examples_per_epoch x 128 matrix # Select triplets based on the embeddings triplets, nrof_random_negs, nrof_triplets = facenet.select_triplets( emb_array, num_per_class, image_data, args.people_per_batch, args.alpha) selection_time = time.time() - start_time print('(nrof_random_negs, nrof_triplets) = (%d, %d): time=%.3f seconds' % ( nrof_random_negs, nrof_triplets, selection_time)) # Perform training on the selected triplets train_time = 0 i = 0 while i * args.batch_size < nrof_triplets * 3 and batch_number < args.epoch_size: start_time = time.time() batch = facenet.get_triplet_batch(triplets, i, args.batch_size) feed_dict = {images_placeholder: batch, learning_rate_placeholder: lr} err, _, step = sess.run([loss, train_op, global_step], feed_dict=feed_dict) if (batch_number % 100 == 0): summary_str, step = sess.run([summary_op, global_step], feed_dict=feed_dict) summary_writer.add_summary(summary_str, global_step=step) duration = time.time() - start_time print('Epoch: [%d][%d/%d]\tTime %.3f\tLoss %2.3f' % (epoch, batch_number+1, args.epoch_size, duration, err)) batch_number += 1 i += 1 train_time += duration # Add validation loss and accuracy to summary summary = tf.Summary() #pylint: disable=maybe-no-member summary.value.add(tag='time/load', simple_value=load_time) summary.value.add(tag='time/selection', simple_value=selection_time) summary.value.add(tag='time/train', simple_value=train_time) summary.value.add(tag='time/total', simple_value=load_time+selection_time+train_time) summary_writer.add_summary(summary, step) return step
def main(args): # 动态import python模块, 这里指的是外部参数指定的网络结构 network = importlib.import_module(args.model_def) subdir = datetime.strftime(datetime.now(), '%Y%m%d-%H%M%S') # os.path.expanduser跨平台支持替换路径中的user路径~ log_dir = os.path.join(os.path.expanduser(args.logs_base_dir), subdir) if not os.path.isdir(log_dir): # Create the log directory if it doesn't exist os.makedirs(log_dir) model_dir = os.path.join(os.path.expanduser(args.models_base_dir), subdir) if not os.path.isdir(model_dir): # Create the model directory if it doesn't exist os.makedirs(model_dir) # Write arguments to a text file facenet.write_arguments_to_file(args, os.path.join(log_dir, 'arguments.txt')) # Store some git revision info in a text file in the log directory src_path, _ = os.path.split(os.path.realpath(__file__)) facenet.store_revision_info(src_path, log_dir, ' '.join(sys.argv)) np.random.seed(seed=args.seed) random.seed(args.seed) # args.data_dir can contain more datasets, separated by comma # TODO: no logic for name conflict? data_dirs = args.data_dir.split(",") train_set = [] for data_dir in data_dirs: if len(data_dir) > 0: train_set.extend(facenet.get_dataset(data_dir)) if args.filter_filename: train_set = filter_dataset(train_set, os.path.expanduser(args.filter_filename), args.filter_percentile, args.filter_min_nrof_images_per_class) nrof_classes = len(train_set) print('Model directory: %s' % model_dir) print('Log directory: %s' % log_dir) pretrained_model = None if args.pretrained_model: pretrained_model = os.path.expanduser(args.pretrained_model) print('Pre-trained model: %s' % pretrained_model) continue_ckpt_dir = None if args.continue_ckpt_dir: continue_ckpt_dir = os.path.expanduser(args.continue_ckpt_dir) print('Continue training from the checkpoint: %s' % continue_ckpt_dir) snapshot_at_step = None if args.snapshot_at_step: snapshot_at_step = int(args.snapshot_at_step) print('Will take a snapshot checkpoint at step', snapshot_at_step) nrof_preprocess_threads = 4 if args.nrof_preprocess_threads: nrof_preprocess_threads = int(args.nrof_preprocess_threads) print('Number of preprocess threads', nrof_preprocess_threads) if args.lfw_dir: print('LFW directory: %s' % args.lfw_dir) # Read the file containing the pairs used for testing pairs = lfw.read_pairs(os.path.expanduser(args.lfw_pairs)) # Get the paths for the corresponding images lfw_paths, actual_issame = lfw.get_paths(os.path.expanduser(args.lfw_dir), pairs, args.lfw_file_ext) with tf.Graph().as_default(): tf.set_random_seed(args.seed) global_step = tf.Variable(0, trainable=False) # Get a list of image paths and their labels image_list, label_list = facenet.get_image_paths_and_labels(train_set) assert len(image_list) > 0, 'The dataset should not be empty' # Create a queue that produces indices into the image_list and label_list # https://www.tensorflow.org/api_guides/python/threading_and_queues # This function converts Python objects of various types to Tensor objects. # It accepts Tensor objects, numpy arrays, Python lists, and Python scalars. labels = ops.convert_to_tensor(label_list, dtype=tf.int32) # This operation returns a 1-D integer tensor representing the shape of input. range_size = array_ops.shape(labels)[0] # Produces the integers from 0 to limit-1 in a queue. index_queue = tf.train.range_input_producer(range_size, num_epochs=None, shuffle=True, seed=None, capacity=32) index_dequeue_op = index_queue.dequeue_many(args.batch_size * args.epoch_size, 'index_dequeue') learning_rate_placeholder = tf.placeholder(tf.float32, name='learning_rate') batch_size_placeholder = tf.placeholder(tf.int32, name='batch_size') phase_train_placeholder = tf.placeholder(tf.bool, name='phase_train') image_paths_placeholder = tf.placeholder(tf.string, shape=(None, 1), name='image_paths') labels_placeholder = tf.placeholder(tf.int64, shape=(None, 1), name='labels') # Creates a queue that dequeues elements in a first-in first-out order. input_queue = data_flow_ops.FIFOQueue(capacity=100000, dtypes=[tf.string, tf.int64], shapes=[(1,), (1,)], shared_name=None, name=None) enqueue_op = input_queue.enqueue_many([image_paths_placeholder, labels_placeholder], name='enqueue_op') # 读取图片文件, 将图片转换成tensor并且做ensembling处理, 结果存入images_and_labels数组 images_and_labels = [] for _ in range(nrof_preprocess_threads): filenames, label = input_queue.dequeue() images = [] # Unpacks the given dimension of a rank-R tensor into rank-(R-1) tensors. for filename in tf.unstack(filenames): file_contents = tf.read_file(filename) # Detects whether an image is a GIF, JPEG, or PNG, and performs the appropriate operation # to convert the input bytes string into a Tensor of type uint8. # Note: decode_gif returns a 4-D array [num_frames, height, width, 3], # as opposed to decode_jpeg and decode_png, which return 3-D arrays [height, width, num_channels]. image = tf.image.decode_image(file_contents, channels=3) # 对训练图片做ensembling # https://www.tensorflow.org/api_docs/python/tf/image # https://www.tensorflow.org/api_docs/python/tf/contrib/image if args.random_rotate: image = tf.py_func(facenet.random_rotate_image, [image], tf.uint8) if args.random_crop: # 训练数据的图片(182)比参数传进来的大小(160)略大, 不做缩放而是直接随机切成160的 image = tf.random_crop(image, [args.image_size, args.image_size, 3]) else: image = tf.image.resize_image_with_crop_or_pad(image, args.image_size, args.image_size) if args.random_flip: image = tf.image.random_flip_left_right(image) if args.random_brightness: image = tf.image.random_brightness(image, max_delta=0.2) # pylint: disable=no-member image.set_shape((args.image_size, args.image_size, 3)) images.append(tf.image.per_image_standardization(image)) images_and_labels.append([images, label]) # Runs a list of tensors to fill a queue to create batches of examples. image_batch, label_batch = tf.train.batch_join( images_and_labels, batch_size=batch_size_placeholder, shapes=[(args.image_size, args.image_size, 3), ()], enqueue_many=True, capacity=4 * nrof_preprocess_threads * args.batch_size, allow_smaller_final_batch=True) # https://stackoverflow.com/questions/34877523/in-tensorflow-what-is-tf-identity-used-for image_batch = tf.identity(image_batch, 'image_batch') image_batch = tf.identity(image_batch, 'input') label_batch = tf.identity(label_batch, 'label_batch') print('Total number of classes: %d' % nrof_classes) print('Total number of examples: %d' % len(image_list)) print('Building training graph') # Build the inference graph prelogits, _ = network.inference(image_batch, args.keep_probability, phase_train=phase_train_placeholder, bottleneck_layer_size=args.embedding_size, weight_decay=args.weight_decay) logits = slim.fully_connected(prelogits, len(train_set), activation_fn=None, weights_initializer=tf.truncated_normal_initializer(stddev=0.1), weights_regularizer=slim.l2_regularizer(args.weight_decay), scope='Logits', reuse=False) # Normalizes along dimension dim using an L2 norm. # For a 1-D tensor with dim = 0, computes output = x / sqrt(max(sum(x**2), epsilon)) # For x with more dimensions, independently normalizes each 1-D slice along dimension dim. # 人脸图片对应的最终编码, 也是算法的核心输出 embeddings = tf.nn.l2_normalize(prelogits, 1, 1e-10, name='embeddings') # Add center loss if args.center_loss_factor > 0.0: prelogits_center_loss, _ = facenet.center_loss(prelogits, label_batch, args.center_loss_alfa, nrof_classes) # Wrapper for Graph.add_to_collection() using the default graph. # Stores value in the collection with the given name. # Note that collections are not sets, so it is possible to add a value to a collection several times. tf.add_to_collection(tf.GraphKeys.REGULARIZATION_LOSSES, # args.center_loss_factor center loss论文里的lambda prelogits_center_loss * args.center_loss_factor) learning_rate = tf.train.exponential_decay(learning_rate_placeholder, global_step, args.learning_rate_decay_epochs * args.epoch_size, args.learning_rate_decay_factor, staircase=True) tf.summary.scalar('learning_rate', learning_rate) # Calculate the average cross entropy loss across the batch cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits( labels=label_batch, logits=logits, name='cross_entropy_per_example') cross_entropy_mean = tf.reduce_mean(cross_entropy, name='cross_entropy') tf.add_to_collection('losses', cross_entropy_mean) # Calculate the total losses regularization_losses = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES) # Adds all input tensors element-wise. total_loss = tf.add_n([cross_entropy_mean] + regularization_losses, name='total_loss') # Build a Graph that trains the model with one batch of examples and updates the model parameters train_op = facenet.train(total_loss, global_step, args.optimizer, learning_rate, args.moving_average_decay, tf.global_variables(), args.log_histograms) # Create a saver saver = tf.train.Saver(tf.trainable_variables(), max_to_keep=3) # Build the summary operation based on the TF collection of Summaries. summary_op = tf.summary.merge_all() # Start running operations on the Graph. gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=args.gpu_memory_fraction) sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options, log_device_placement=False)) sess.run(tf.global_variables_initializer()) sess.run(tf.local_variables_initializer()) summary_writer = tf.summary.FileWriter(log_dir, sess.graph) coord = tf.train.Coordinator() tf.train.start_queue_runners(coord=coord, sess=sess) with sess.as_default(): if pretrained_model: print('Restoring pretrained model: %s' % pretrained_model) saver.restore(sess, pretrained_model) elif continue_ckpt_dir: files = os.listdir(continue_ckpt_dir) meta_files = [s for s in files if s.endswith('.meta')] if len(meta_files) == 0: raise ValueError('No meta file found in %s' % continue_ckpt_dir) elif len(meta_files) > 1: raise ValueError( 'There should not be more than one meta file in %s' % continue_ckpt_dir) saver = tf.train.import_meta_graph(continue_ckpt_dir + "/" + meta_files[0]) latest_checkpoint = tf.train.latest_checkpoint(continue_ckpt_dir) print('Restoring checkpoint: %s' % latest_checkpoint) saver.restore(sess, latest_checkpoint) # TODO: don't know why global_step is not saved. get it from the filename last_step = int(os.path.basename(latest_checkpoint).split('-')[-1]) print('Checkpoint restored, last step is ', str(last_step)) sess.run(global_step.assign(last_step)) # Training and validation loop print('Running training') epoch = 0 while epoch < args.max_nrof_epochs: step = sess.run(global_step, feed_dict=None) epoch_size = args.epoch_size epoch = step // epoch_size if args.learning_rate > 0.0: lr = args.learning_rate else: # Read the schedule file each epoch, you can change the file content during running lr = facenet.get_learning_rate_from_file(args.learning_rate_schedule_file, epoch) # Special value means stop if lr == 0.0: break # Train for one epoch train(args, sess, epoch, image_list, label_list, index_dequeue_op, enqueue_op, image_paths_placeholder, labels_placeholder, learning_rate_placeholder, phase_train_placeholder, batch_size_placeholder, global_step, total_loss, train_op, summary_op, summary_writer, regularization_losses, lr, snapshot_at_step, saver, model_dir, subdir ) # Save variables and the metagraph if it doesn't exist already (step in filename is the next step after restore) save_variables_and_metagraph(sess, saver, summary_writer, model_dir, subdir, step + epoch_size) # Evaluate on LFW if args.lfw_dir and args.lfw_epoch_interval > 0: if epoch % args.lfw_epoch_interval == 0: evaluate(sess, enqueue_op, image_paths_placeholder, labels_placeholder, phase_train_placeholder, batch_size_placeholder, embeddings, label_batch, lfw_paths, actual_issame, args.lfw_batch_size, args.lfw_nrof_folds, log_dir, step, summary_writer) # Print current time print("Current date time:", datetime.strftime(datetime.now(), '%Y%m%d-%H%M%S')) return model_dir
def train(args, sess, dataset, epoch, image_paths_placeholder, labels_placeholder, labels_batch, batch_size_placeholder, learning_rate_placeholder, phase_train_placeholder, enqueue_op, input_queue, global_step, embeddings, total_loss, reg_loss, train_op, summary_op, summary_writer, learning_rate_schedule_file, embedding_size): batch_number = 0 if args.learning_rate>0.0: lr = args.learning_rate else: lr = facenet.get_learning_rate_from_file(learning_rate_schedule_file, epoch) while batch_number < args.epoch_size: # Sample people randomly from the dataset image_paths, num_per_class = sample_people(dataset, args.people_per_batch,args.images_per_person) print('Running forward pass on sampled images: ', end='') start_time = time.time() nrof_examples = args.people_per_batch * args.images_per_person labels_array = np.reshape(np.arange(nrof_examples),(-1,3)) image_paths_array = np.reshape(np.expand_dims(np.array(image_paths),1), (-1,3)) sess.run(enqueue_op, {image_paths_placeholder: image_paths_array, labels_placeholder: labels_array}) emb_array = np.zeros((nrof_examples, embedding_size)) nrof_batches = int(np.ceil(nrof_examples / args.batch_size)) for i in range(nrof_batches): batch_size = min(nrof_examples-i*args.batch_size, args.batch_size) emb, lab = sess.run([embeddings, labels_batch], feed_dict={batch_size_placeholder: batch_size, learning_rate_placeholder: lr, phase_train_placeholder: True}) emb_array[lab,:] = emb print('%.3f' % (time.time()-start_time)) # Select triplets based on the embeddings print('Selecting suitable triplets for training') triplets, nrof_random_negs, nrof_triplets = select_triplets(emb_array, num_per_class, image_paths, args.people_per_batch, args.alpha) selection_time = time.time() - start_time print('(nrof_random_negs, nrof_triplets) = (%d, %d): time=%.3f seconds' % (nrof_random_negs, nrof_triplets, selection_time)) # Perform training on the selected triplets nrof_batches = int(np.ceil(nrof_triplets*3/args.batch_size)) print('选择出来的 triplets 形成的 batch 有多少个:', nrof_batches) triplet_paths = list(itertools.chain(*triplets)) labels_array = np.reshape(np.arange(len(triplet_paths)),(-1,3)) triplet_paths_array = np.reshape(np.expand_dims(np.array(triplet_paths),1), (-1,3)) sess.run(enqueue_op, {image_paths_placeholder: triplet_paths_array, labels_placeholder: labels_array}) nrof_examples = len(triplet_paths) train_time = 0 i = 0 emb_array = np.zeros((nrof_examples, args.embedding_size)) loss_array = np.zeros((nrof_triplets,)) summary = tf.Summary() step = 0 while i < nrof_batches: start_time = time.time() batch_size = min(nrof_examples-i*args.batch_size, args.batch_size) feed_dict = {batch_size_placeholder: batch_size, learning_rate_placeholder: lr, phase_train_placeholder: True} total_err, reg_err, _, step, emb, lab = sess.run([total_loss, reg_loss, train_op, global_step, embeddings, labels_batch], feed_dict=feed_dict) duration = time.time() - start_time print('Epoch: [%d][%d/%d]\tTime %.3f\tTotal Loss %2.3f\tReg Loss %2.3f, lr %2.5f' % (epoch, batch_number+2, args.epoch_size, duration, total_err, reg_err, lr)) batch_number += 2 i += 1 train_time += duration return step
def train(args, sess, dataset_ID, dataset_camera, epoch, image_paths_placeholder_ID, image_paths_placeholder_camera, labels_placeholder_ID, labels_placeholder_camera, labels_batch_ID, labels_batch_camera, batch_size_placeholder, learning_rate_placeholder, phase_train_placeholder, enqueue_op_ID, enqueue_op_camera, global_step, embeddings_ID, embeddings_camera, loss_total, triplet_loss_ID, triplet_loss_camera, loss_feature_map1, loss_feature_map2, loss_feature_map3, regularization_losses, train_op, summary_writer, learning_rate_schedule_file, embedding_size, feature_map1_ID, feature_map1_camera): batch_number = 0 if args.learning_rate > 0.0: lr = args.learning_rate else: lr = facenet.get_learning_rate_from_file(learning_rate_schedule_file, epoch) while batch_number < args.epoch_size: # Sample people randomly from the dataset image_paths_ID, num_per_class_ID = sample_people( dataset_ID, args.people_per_batch, args.images_per_person) image_paths_camera, num_per_class_camera = sample_people( dataset_camera, args.people_per_batch, args.images_per_person) print('Running forward pass on sampled images: ', end='') start_time = time.time() nrof_examples = args.people_per_batch * args.images_per_person labels_array_ID = np.reshape(np.arange(nrof_examples), (-1, 3)) labels_array_camera = np.reshape(np.arange(nrof_examples), (-1, 3)) image_paths_array_ID = np.reshape( np.expand_dims(np.array(image_paths_ID), 1), (-1, 3)) image_paths_array_camera = np.reshape( np.expand_dims(np.array(image_paths_camera), 1), (-1, 3)) sess.run( enqueue_op_ID, { image_paths_placeholder_ID: image_paths_array_ID, labels_placeholder_ID: labels_array_ID }) sess.run( enqueue_op_camera, { image_paths_placeholder_camera: image_paths_array_camera, labels_placeholder_camera: labels_array_camera }) emb_array_ID = np.zeros((nrof_examples, embedding_size)) emb_array_camera = np.zeros((nrof_examples, embedding_size)) nrof_batches = int(np.ceil(nrof_examples / args.batch_size)) for i in range(nrof_batches): batch_size = min(nrof_examples - i * args.batch_size, args.batch_size) emb_ID, lab_ID = sess.run( [embeddings_ID, labels_batch_ID], feed_dict={ batch_size_placeholder: batch_size, learning_rate_placeholder: lr, phase_train_placeholder: True }) emb_array_ID[lab_ID, :] = emb_ID emb_camera, lab_camera = sess.run( [embeddings_camera, labels_batch_camera], feed_dict={ batch_size_placeholder: batch_size, learning_rate_placeholder: lr, phase_train_placeholder: True }) emb_array_camera[lab_camera, :] = emb_camera print('%.3f' % (time.time() - start_time)) # Select triplets based on the embeddings print('Selecting suitable triplets for training') triplets_ID, nrof_random_negs_ID, nrof_triplets_ID = select_triplets( emb_array_ID, num_per_class_ID, image_paths_ID, args.people_per_batch, args.alpha) triplets_camera, nrof_random_negs_camera, nrof_triplets_camera = select_triplets( emb_array_camera, num_per_class_camera, image_paths_camera, args.people_per_batch, args.alpha) nrof_triplets = min(nrof_triplets_ID, nrof_triplets_camera) nrof_triplets_ID = nrof_triplets nrof_triplets_camera = nrof_triplets triplets_ID_new = triplets_ID[0:nrof_triplets] triplets_camera_new = triplets_camera[0:nrof_triplets] selection_time = time.time() - start_time print( '(nrof_random_negs, nrof_triplets) of ID = ( %d, %d): time=%.3f seconds' % (nrof_random_negs_ID, nrof_triplets_ID, selection_time)) print( '(nrof_random_negs, nrof_triplets) of camera = ( %d, %d): time=%.3f seconds' % (nrof_random_negs_camera, nrof_triplets_camera, selection_time)) # Perform training on the selected triplets nrof_batches_ID = int(np.ceil(nrof_triplets_ID * 3 / args.batch_size)) nrof_batches_camera = int( np.ceil(nrof_triplets_camera * 3 / args.batch_size)) nrof_batches = min(nrof_batches_camera, nrof_batches_ID) triplet_paths_ID = list(itertools.chain(*triplets_ID_new)) triplet_paths_camera = list(itertools.chain(*triplets_camera_new)) labels_array_ID = np.reshape(np.arange(len(triplet_paths_ID)), (-1, 3)) labels_array_camera = np.reshape(np.arange(len(triplet_paths_camera)), (-1, 3)) triplet_paths_array_ID = np.reshape( np.expand_dims(np.array(triplet_paths_ID), 1), (-1, 3)) triplet_paths_array_camera = np.reshape( np.expand_dims(np.array(triplet_paths_camera), 1), (-1, 3)) sess.run( enqueue_op_ID, { image_paths_placeholder_ID: triplet_paths_array_ID, labels_placeholder_ID: labels_array_ID }) sess.run( enqueue_op_camera, { image_paths_placeholder_camera: triplet_paths_array_camera, labels_placeholder_camera: labels_array_camera }) nrof_examples_ID = len(triplet_paths_ID) nrof_examples_camera = len(triplet_paths_camera) nrof_examples = min(nrof_examples_ID, nrof_examples_camera) train_time = 0 i = 0 summary = tf.Summary() while i < nrof_batches: start_time = time.time() batch_size = min(nrof_examples - i * args.batch_size, args.batch_size) feed_dict = { batch_size_placeholder: batch_size, learning_rate_placeholder: lr, phase_train_placeholder: True } # err_feature_map1 = sess.run([loss_feature_map1],feed_dict=feed_dict) err_total, err_ID, err_camera, err_feature_map1, err_feature_map2, err_feature_map3, err_regularization, _, step, feature_map1ID, feature_map1camera = sess.run( [ loss_total, triplet_loss_ID, triplet_loss_camera, loss_feature_map1, loss_feature_map2, loss_feature_map3, regularization_losses, train_op, global_step, feature_map1_ID, feature_map1_camera ], feed_dict=feed_dict) duration = time.time() - start_time print('Epoch: [%d][%d/%d]\tTime %.3f\tLoss %2.3f' % (epoch, batch_number + 1, args.epoch_size, duration, err_total)) print( 'triplet_loss_ID: %2.3f\ttriplet_loss_camera:%2.3f\tloss_feature_map1: %f\tloss_feature_map2:%f\tloss_feature_map3:%f' % (err_ID, err_camera, err_feature_map1, err_feature_map2, err_feature_map3)) # print("regularization_losses is :", err_regularization) batch_number += 1 i += 1 train_time += duration summary.value.add(tag='loss', simple_value=err_total) # Add validation loss and accuracy to summary #pylint: disable=maybe-no-member summary.value.add(tag='time/selection', simple_value=selection_time) summary_writer.add_summary(summary) return step
def train(args, sess, dataset, epoch, image_paths_placeholder, labels_placeholder, labels_batch, batch_size_placeholder, learning_rate_placeholder, phase_train_placeholder, enqueue_op, input_queue, global_step, embeddings, loss, train_op, summary_op, summary_writer, learning_rate_schedule_file, embedding_size, anchor, positive, negative, triplet_loss): import imgaug import imgaug.augmenters as iaa batch_number = 0 augmentation = iaa.Sequential([iaa.Rotate((-170, 170))]) if args.learning_rate > 0.0: lr = args.learning_rate else: lr = facenet.get_learning_rate_from_file(learning_rate_schedule_file, epoch) while batch_number < args.epoch_size: # Sample people randomly from the dataset image_paths, num_per_class = sample_people(dataset, args.people_per_batch, args.images_per_person) print('Running forward pass on sampled images: ', end='') start_time = time.time() nrof_examples = args.people_per_batch * args.images_per_person labels_array = np.reshape(np.arange(nrof_examples), (-1, 3)) image_paths_array = np.reshape( np.expand_dims(np.array(image_paths), 1), (-1, 3)) images_array = [] for p in image_paths_array: imgs = [] for path in p: image = cv2.imread(path) w, h, c = image.shape r = math.ceil((w**2 + h**2)**0.5) pw = math.ceil((r - w) / 2) ph = math.ceil((r - h) / 2) image = np.pad(image, ((pw, pw), (ph, ph), (0, 0))) m = path[:-4] + '.npy' with open(m, 'rb') as file: mask = np.load(file) mask = np.pad(mask, ((pw, pw), (ph, ph))) # Augmenters that are safe to apply to masks # Some, such as Affine, have settings that make them unsafe, so always # test your augmentation on masks MASK_AUGMENTERS = [ "Sequential", "SomeOf", "OneOf", "Sometimes", "Fliplr", "Flipud", "CropAndPad", "Affine", "PiecewiseAffine" ] def hook(images, augmenter, parents, default): """Determines which augmenters to apply to masks.""" return augmenter.__class__.__name__ in MASK_AUGMENTERS # Store shapes before augmentation to compare image_shape = image.shape mask_shape = mask.shape # Make augmenters deterministic to apply similarly to images and masks det = augmentation.to_deterministic() #image = det.augment_image(image) #mask = det.augment_image(mask) # Verify that shapes didn't change assert image.shape == image_shape, "Augmentation shouldn't change image size" assert mask.shape == mask_shape, "Augmentation shouldn't change mask size" horizontal_indicies = np.where(np.any(mask, axis=0))[0] vertical_indicies = np.where(np.any(mask, axis=1))[0] x1, x2 = horizontal_indicies[[0, -1]] y1, y2 = vertical_indicies[[0, -1]] x2 += 1 y2 += 1 mask_out = image * np.stack([mask, mask, mask], axis=2) out = mask_out[y1:y2, x1:x2] out = cv2.resize(out, dsize=(args.image_size, args.image_size)) #cv2.imwrite('verify.png', out) imgs.append(out) images_array.append(imgs) images_array = np.array(images_array) sess.run( enqueue_op, { # image_paths_placeholder: image_paths_array, image_paths_placeholder: images_array, labels_placeholder: labels_array }) emb_array = np.zeros((nrof_examples, embedding_size)) nrof_batches = int(np.ceil(nrof_examples / args.batch_size)) for i in range(nrof_batches): batch_size = min(nrof_examples - i * args.batch_size, args.batch_size) emb, lab = sess.run( [embeddings, labels_batch], feed_dict={ batch_size_placeholder: batch_size, learning_rate_placeholder: lr, phase_train_placeholder: True }) emb_array[lab, :] = emb print('%.3f' % (time.time() - start_time)) # Select triplets based on the embeddings print('Selecting suitable triplets for training') triplets, nrof_random_negs, nrof_triplets = select_triplets( emb_array, num_per_class, np.reshape(images_array, (-1, args.image_size, args.image_size, 3)), args.people_per_batch, args.alpha) selection_time = time.time() - start_time print( '(nrof_random_negs, nrof_triplets) = (%d, %d): time=%.3f seconds' % (nrof_random_negs, nrof_triplets, selection_time)) # Perform training on the selected triplets nrof_batches = int(np.ceil(nrof_triplets * 3 / args.batch_size)) triplet_paths = list(itertools.chain(*triplets)) labels_array = np.reshape(np.arange(len(triplet_paths)), (-1, 3)) triplet_paths_array = np.reshape( np.expand_dims(np.array(triplet_paths), 1), (-1, 3, args.image_size, args.image_size, 3)) sess.run( enqueue_op, { image_paths_placeholder: triplet_paths_array, labels_placeholder: labels_array }) nrof_examples = len(triplet_paths) train_time = 0 i = 0 emb_array = np.zeros((nrof_examples, embedding_size)) loss_array = np.zeros((nrof_triplets, )) summary = tf.compat.v1.Summary() step = 0 while i < nrof_batches: start_time = time.time() batch_size = min(nrof_examples - i * args.batch_size, args.batch_size) feed_dict = { batch_size_placeholder: batch_size, learning_rate_placeholder: lr, phase_train_placeholder: True } err, _, step, emb, lab = sess.run( [loss, train_op, global_step, embeddings, labels_batch], feed_dict=feed_dict) emb_array[lab, :] = emb loss_array[i] = err duration = time.time() - start_time print('Epoch: [%d][%d/%d]\tTime %.3f\tLoss %2.3f' % (epoch, batch_number + 1, args.epoch_size, duration, err)) batch_number += 1 i += 1 train_time += duration summary.value.add(tag='loss', simple_value=err) # Add validation loss and accuracy to summary #pylint: disable=maybe-no-member summary.value.add(tag='time/selection', simple_value=selection_time) summary_writer.add_summary(summary, step) return step
def triplet(args, sess, images_list, epoch, image_placeholder, code_placehoder, batch_size_placeholder, learning_rate_placeholder, phase_train_placeholder, global_step, embeddings, loss, train_op, summary_op, summary_writer, learning_rate_schedule_file, embedding_size, anchor, positive, negative, triplet_loss, code): batch_number = 0 images_data = h5py.File('../face_hdf5_150by150/train_classification.h5') images = images_data['data'] image_list_p = images_list[0] image_list_n = images_list[1] if args.learning_rate>0.0: lr = args.learning_rate else: lr = facenet.get_learning_rate_from_file(learning_rate_schedule_file, epoch) random.shuffle(image_list_n) start_time = time.time() nrof_examples = len(image_list_p) # sess.run(enqueue_op, {image_paths_placeholder: image_paths_array, labels_placeholder: labels_array}) emb_p_array = np.zeros((0, embedding_size)) emb_n_array = np.zeros((0, embedding_size)) nrof_batches = int(np.ceil(nrof_examples / args.batch_size)) image_mean = cv2.imread('../image_mean1.jpg') image_mean = cv2.resize(image_mean, (120,120)) image_mean = image_mean.astype(np.float32) print('load mean image done!') print('start to select positive') for i in range(nrof_batches): batch_number = i print(i) # batch_size = min(nrof_examples-i*args.batch_size, args.batch_size) batch_size = args.batch_size [image_p_array, image_list_p]= facenet.get_image_batch(images, image_list_p, batch_size, batch_number, image_mean, args) code_batch = code[0:batch_size] emb_p= sess.run([embeddings], feed_dict={batch_size_placeholder: batch_size, learning_rate_placeholder: lr, phase_train_placeholder: True, image_placeholder: image_p_array, code_placehoder: code_batch}) emb_p_array = np.vstack((emb_p_array,emb_p[0])) print('number of positive batch:', i+1) nrof_n_examples = min(nrof_examples, 30000) nrof_n_batches = int(np.ceil(nrof_n_examples / args.batch_size)) print('start to select negative') for j in range(nrof_n_batches): batch_number = j # batch_size = min(nrof_examples-j*args.batch_size, args.batch_size) print(j) batch_size = args.batch_size [image_n_array, image_list_n] = facenet.get_image_batch(images, image_list_n, batch_size, batch_number, image_mean, args) code_batch = code[0:batch_size] emb_n = sess.run([embeddings], feed_dict={batch_size_placeholder: batch_size, learning_rate_placeholder: lr, phase_train_placeholder: True, image_placeholder: image_n_array, code_placehoder: code_batch}) emb_n_array = np.vstack((emb_n_array,emb_n[0])) print('number of negative batch:', j+1) print('compute feature is done!') print('%.3f' % (time.time()-start_time)) # Select triplets based on the embeddings print('Selecting suitable triplets for training') triplets, nrof_random_negs, nrof_triplets = select_binary_triplets(emb_p_array, emb_n_array, image_list_p, image_list_n, args.alpha) selection_time = time.time() - start_time print('(nrof_random_negs, nrof_triplets) = (%d, %d): time=%.3f seconds' % (nrof_random_negs, nrof_triplets, selection_time)) return triplets
def train(args, sess, dataset, epoch, image_paths_placeholder, labels_placeholder, labels_batch, batch_size_placeholder, learning_rate_placeholder, phase_train_placeholder, enqueue_op, input_queue, global_step, embeddings, loss, train_op, summary_op, summary_writer, learning_rate_schedule_file, embedding_size, anchor, positive, negative, triplet_loss): batch_number = 0 if args.learning_rate > 0.0: lr = args.learning_rate else: lr = facenet.get_learning_rate_from_file(learning_rate_schedule_file, epoch) #下面的一个while循环运行一个批处理 while batch_number < args.epoch_size: # Sample people randomly from the dataset #默认下sample返回了1800,即1800个图像 image_paths, num_per_class = sample_people(dataset, args.people_per_batch, args.images_per_person) #以上的输出里很多都是一张图片,那么这种肯定没有办法作为a和p,程序是怎么筛选出去的呢? print('Running forward pass on sampled images: ', end='') start_time = time.time() nrof_examples = args.people_per_batch * args.images_per_person # 根据默认参数,将输入的1800个图像变成了600*3的二维数据,如果不够1800张图像,可能要修改这里 labels_array = np.reshape(np.arange(nrof_examples), (-1, 3)) #reshape(a,(-1,3))表示只给定列数为3,行数自行算出 image_paths_array = np.reshape( np.expand_dims(np.array(image_paths), 1), (-1, 3)) #这里开辟了一个新的线程用于在内存里读取数据 sess.run( enqueue_op, { image_paths_placeholder: image_paths_array, labels_placeholder: labels_array }) # 默认的参数中,nrof_examples是1800,embedding_size是128 emb_array = np.zeros((nrof_examples, embedding_size)) # nrof_batches,默认是1800/90=20 nrof_batches = int(np.ceil(nrof_examples / args.batch_size)) # 批处理求特征,默认为20个批 for i in range(nrof_batches): batch_size = min(nrof_examples - i * args.batch_size, args.batch_size) emb, lab = sess.run( [embeddings, labels_batch], feed_dict={ batch_size_placeholder: batch_size, learning_rate_placeholder: lr, phase_train_placeholder: True }) emb_array[lab, :] = emb print('%.3f' % (time.time() - start_time)) # Select triplets based on the embeddings print('Selecting suitable triplets for training') triplets, nrof_random_negs, nrof_triplets = select_triplets( emb_array, num_per_class, image_paths, args.people_per_batch, args.alpha) selection_time = time.time() - start_time print( '(nrof_random_negs, nrof_triplets) = (%d, %d): time=%.3f seconds' % (nrof_random_negs, nrof_triplets, selection_time)) # Perform training on the selected triplets nrof_batches = int(np.ceil(nrof_triplets * 3 / args.batch_size)) triplet_paths = list(itertools.chain(*triplets)) labels_array = np.reshape(np.arange(len(triplet_paths)), (-1, 3)) triplet_paths_array = np.reshape( np.expand_dims(np.array(triplet_paths), 1), (-1, 3)) # 读取数据的操作 sess.run( enqueue_op, { image_paths_placeholder: triplet_paths_array, labels_placeholder: labels_array }) nrof_examples = len(triplet_paths) train_time = 0 i = 0 emb_array = np.zeros((nrof_examples, embedding_size)) loss_array = np.zeros((nrof_triplets, )) # 根据求出的特征计算triplet损失函数并进行优化 summary = tf.Summary() while i < nrof_batches: start_time = time.time() batch_size = min(nrof_examples - i * args.batch_size, args.batch_size) feed_dict = { batch_size_placeholder: batch_size, learning_rate_placeholder: lr, phase_train_placeholder: True } #sess run 有5个输入,fetches,先运行loss。前向计算的损失,train_op是根据损失来计算梯度,来对参数进行优化 err, _, step, emb, lab = sess.run( [loss, train_op, global_step, embeddings, labels_batch], feed_dict=feed_dict) emb_array[lab, :] = emb loss_array[i] = err duration = time.time() - start_time print('Epoch: [%d][%d/%d]\tTime %.3f\tLoss %2.3f' % (epoch, batch_number + 1, args.epoch_size, duration, err)) batch_number += 1 i += 1 train_time += duration summary.value.add(tag='loss', simple_value=err) # Add validation loss and accuracy to summary #pylint: disable=maybe-no-member summary.value.add(tag='time/selection', simple_value=selection_time) summary_writer.add_summary(summary, step) return step
def train(args, sess, epoch, image_list, label_list, index_dequeue_op, enqueue_op, image_paths_placeholder, labels_placeholder, learning_rate_placeholder, phase_train_placeholder, batch_size_placeholder, global_step, loss, train_op, summary_op, summary_writer, regularization_losses, learning_rate_schedule_file \ ,test_image_list, test_label_list, test_index_dequeue_op, test_enqueue_op, image_batch, label_batch): batch_number = 0 if args.learning_rate > 0.0: lr = args.learning_rate else: lr = facenet.get_learning_rate_from_file(learning_rate_schedule_file, epoch) print("Epoch: [%d]\tLr: %f" % (epoch, lr)) index_epoch = sess.run(index_dequeue_op) label_epoch = np.array(label_list)[index_epoch] image_epoch = np.array(image_list)[index_epoch] # Enqueue one epoch of image paths and labels labels_array = np.expand_dims(np.array(label_epoch), 1) image_paths_array = np.expand_dims(np.array(image_epoch), 1) sess.run( enqueue_op, { image_paths_placeholder: image_paths_array, labels_placeholder: labels_array }) # Training loop train_time = 0 while batch_number < args.epoch_size: start_time = time.time() feed_dict = { learning_rate_placeholder: lr, phase_train_placeholder: True, batch_size_placeholder: args.batch_size } if ((batch_number) % 100 == 0): err, _, step, reg_loss, summary_str = sess.run([ loss, train_op, global_step, regularization_losses, summary_op ], feed_dict=feed_dict) summary_writer.add_summary(summary_str, global_step=step) #val val_err,reg_err=val_loss(args, sess, loss,regularization_losses, epoch, step, test_image_list, test_label_list, test_index_dequeue_op,\ image_batch, label_batch, phase_train_placeholder) summary = tf.Summary() summary.value.add(tag='val/total_loss', simple_value=val_err) summary.value.add(tag='val/reg_loss', simple_value=reg_err) summary_writer.add_summary(summary, global_step=step) else: err, _, step, reg_loss = sess.run( [loss, train_op, global_step, regularization_losses], feed_dict=feed_dict) duration = time.time() - start_time print('Epoch: [%d][%d/%d]\tTime %.3f\tLoss %2.3f\tRegLoss %2.3f' % (epoch, batch_number + 1, args.epoch_size, duration, err, np.sum(reg_loss))) batch_number += 1 train_time += duration # Add validation loss and accuracy to summary summary = tf.Summary() #pylint: disable=maybe-no-member summary.value.add(tag='time/total', simple_value=train_time) summary_writer.add_summary(summary, step) #Validate loop # batch_number=0 # test_index_epoch = sess.run(test_index_dequeue_op) # test_label_epoch = np.array(test_label_list)[test_index_epoch] # test_image_epoch = np.array(test_image_list)[test_index_epoch] # # Enqueue one epoch of image paths and labels # test_labels_array = np.expand_dims(np.array(test_label_epoch), 1) # test_image_paths_array = np.expand_dims(np.array(test_image_epoch), 1) # sess.run(enqueue_op, {image_paths_placeholder: test_image_paths_array, labels_placeholder: test_labels_array}) # val_train_time = 0 # err_sum=0 # while batch_number < test_epoch_size: # start_time = time.time() # feed_dict = {learning_rate_placeholder: lr, phase_train_placeholder: False, # batch_size_placeholder: test_batch_size} # err = sess.run([loss], feed_dict=feed_dict) # err=np.mean(err) # err_sum+=err # duration = time.time() - start_time # print('Epoch: [%d][%d/%d]\tTime %.3f\tTest_Loss %2.3f\t' % # (epoch, batch_number + 1, test_batch_num, duration, err)) # batch_number += 1 # val_train_time += duration # # Add validation loss and accuracy to summary # summary = tf.Summary() # summary.value.add(tag='val/loss', simple_value=(err_sum/test_batch_num)) # summary_writer.add_summary(summary, global_step=step) # summary = tf.Summary() # summary.value.add(tag='val_time/total', simple_value=val_train_time) # summary_writer.add_summary(summary, step) return step
def train_multi_gpu(args, sess, dataset, epoch, image_paths_placeholder, labels_placeholder, labels_batch, batch_size_placeholder, learning_rate_placeholder, phase_train_placeholder, enqueue_op, input_queue, global_step, embeddings, loss, train_op, summary_op, summary_writer, learning_rate_schedule_file, embedding_size): batch_number = 0 if args.learning_rate > 0.0: lr = args.learning_rate else: lr = facenet.get_learning_rate_from_file(learning_rate_schedule_file, epoch) while batch_number < args.epoch_size: # Sample people randomly from the dataset image_paths, num_per_class = sample_people(dataset, args.people_per_batch, args.images_per_person) print('Running forward pass on sampled images: ', end='') start_time = time.time() nrof_examples = args.people_per_batch * args.images_per_person labels_array = np.reshape(np.arange(nrof_examples), (-1, 3)) image_paths_array = np.reshape( np.expand_dims(np.array(image_paths), 1), (-1, 3)) sess.run( enqueue_op, { image_paths_placeholder: image_paths_array, labels_placeholder: labels_array }) emb_array = np.zeros((nrof_examples, embedding_size)) nrof_batches = int(np.ceil(nrof_examples / args.batch_size)) for i in range(nrof_batches): batch_size = min(nrof_examples - i * args.batch_size, args.batch_size) # ------ new start----- # for j in range(args.num_gpus): # emb_tmp, lab_tmp = sess.run([embeddings_split, labels_batch_split], feed_dict={batch_size_placeholder: batch_size, # learning_rate_placeholder: lr, phase_train_placeholder: True}) # emb_array[lab_tmp,:] = emb_tmp # ------ new end -------- # ------ old start ------ emb, lab = sess.run( [embeddings, labels_batch], feed_dict={ batch_size_placeholder: batch_size, learning_rate_placeholder: lr, phase_train_placeholder: True }) emb_array[lab, :] = emb # ------ old end -------- print('%.3f' % (time.time() - start_time)) # Select triplets based on the embeddings print('Selecting suitable triplets for training') triplets, nrof_random_negs, nrof_triplets = select_triplets( emb_array, num_per_class, image_paths, args.people_per_batch, args.alpha) selection_time = time.time() - start_time print( '(nrof_random_negs, nrof_triplets) = (%d, %d): time=%.3f seconds' % (nrof_random_negs, nrof_triplets, selection_time)) # Perform training on the selected triplets nrof_batches = int(np.ceil(nrof_triplets * 3 / args.batch_size)) print('选择出来的 triplets 形成的 batch 有多少个:', nrof_batches) triplet_paths = list(itertools.chain(*triplets)) labels_array = np.reshape(np.arange(len(triplet_paths)), (-1, 3)) triplet_paths_array = np.reshape( np.expand_dims(np.array(triplet_paths), 1), (-1, 3)) sess.run( enqueue_op, { image_paths_placeholder: triplet_paths_array, labels_placeholder: labels_array }) nrof_examples = len(triplet_paths) train_time = 0 i = 0 emb_array = np.zeros((nrof_examples, args.embedding_size)) loss_array = np.zeros((nrof_triplets, )) summary = tf.Summary() step = 0 while i < nrof_batches: start_time = time.time() batch_size = min(nrof_examples - i * args.batch_size, args.batch_size) feed_dict = { batch_size_placeholder: batch_size, learning_rate_placeholder: lr, phase_train_placeholder: True } # ---- new start ----- # total_err_split = [] # reg_err_split = [] # emb_split_new = [] # lab_split_new = [] # for j in range(args.num_gpus): # if j % 2 == 1: # total_err_tmp, reg_err_tmp, _, step, emb_new_tmp, lab_new_tmp = sess.run([loss['total_loss'], loss['total_reg'], train_op, global_step, embeddings_split[j], labels_batch_split[j]], feed_dict=feed_dict) # else: # total_err_tmp, reg_err_tmp, _, emb_new_tmp, lab_new_tmp = sess.run([loss['total_loss'], loss['total_reg'], train_op, embeddings_split[j], labels_batch_split[j]], feed_dict=feed_dict) # total_err_split.append(total_err_tmp) # reg_err_split.append(reg_err_tmp) # emb_split_new.append(emb_new_tmp) # lab_split_new(lab_new_tmp) # total_err = tf.reduce_mean(total_err_split) # reg_err = tf.reduce_mean(reg_err_split) # 将 tensor 转换为 float # total_err_float = tf.to_float(total_err, name='totensor_1') # reg_err_float = tf.to_float(reg_err, name='totensor_2') # emb = tf.reduce_mean(emb_split_new, 0) # lab = tf.reduce_mean(lab_split_new, 0) # ---- new end ------- # ----- old start ----- total_err, reg_err, _, step, emb, lab = sess.run( [ loss['total_loss'], loss['total_reg'], train_op, global_step, embeddings, labels_batch ], feed_dict=feed_dict) # ----- old end ------- duration = time.time() - start_time # print('SSSSSSSSSSSSSSSStep======', step) print( 'Epoch: [%d][%d/%d]\tTime %.3f\tTotal Loss %2.3f\tReg Loss %2.3f, lr %2.5f' % (epoch, batch_number + 1, args.epoch_size, duration, total_err, reg_err, lr)) batch_number += 1 i += 1 train_time += duration return step