def train(self,
X_train,
y_train,
X_valid,
y_valid,
batch_size=50,
alpha=0.001,
lmbda=0.0001,
num_epochs=10):
m, n = X_train.shape
num_batches = m // batch_size
report = "{:3d}: training loss = {:.2f} | validation loss = {:.2f}"
losses = []
for epoch in range(num_epochs):
train_loss = 0.0
for _ in range(num_batches):
W1, b1 = self.params['W1'], self.params['b1']
W2, b2 = self.params['W2'], self.params['b2']
# select a random mini-batch
batch_idx = np.random.choice(m, batch_size, replace=False)
X_batch, y_batch = X_train[batch_idx], y_train[batch_idx]
# train on mini-batch
data_loss, gradient = self.train_step(X_batch, y_batch)
reg_loss = 0.5 * (np.sum(W1**2) + np.sum(W2**2))
train_loss += (data_loss + lmbda * reg_loss)
losses.append(data_loss + lmbda * reg_loss)
# regularization
gradient['W1'] += lmbda * W1
gradient['W2'] += lmbda * W2
# update parameters
for p in self.params:
self.params[p] = self.params[p] - alpha * gradient[p]
# report training loss and validation loss
train_loss /= num_batches
valid_loss = softmax_loss(self.forward(X_valid),
y_valid,
mode='test')
print(report.format(epoch + 1, train_loss, valid_loss))
return losses
def train_step(self, X, y):
W1, b1 = self.params['W1'], self.params['b1']
W2, b2 = self.params['W2'], self.params['b2']
# forward step
h_in = X @ W1 + b1 # hidden layer input
h = np.maximum(0, h_in) # hidden layer output (using ReLU)
scores = h @ W2 + b2 # neural net output
#print("scores values is {} ".format(scores))
# compute loss
loss, dscores = softmax_loss(scores, y)
# backward step
db2 = dscores.sum(axis=0)
dW2 = h.T @ dscores
dh = dscores @ W2.T
dh[h_in < 0] = 0.0
db1 = dh.sum(axis=0)
dW1 = X.T @ dh
gradient = {'W1': dW1, 'b1': db1, 'W2': dW2, 'b2': db2}
return loss, gradient
def main(args):
#network = importlib.import_module(args.model_def)
subdir = datetime.strftime(datetime.now(), '%Y%m%d-%H%M%S')
log_dir = os.path.join(os.path.expanduser(args.logs_base_dir), subdir)
if not os.path.isdir(
log_dir): # Create the log directory if it doesn't exist
os.makedirs(log_dir)
model_dir = os.path.join(os.path.expanduser(args.models_base_dir), subdir)
if not os.path.isdir(
model_dir): # Create the model directory if it doesn't exist
os.makedirs(model_dir)
# Write arguments to a text file
utils.write_arguments_to_file(args, os.path.join(log_dir, 'arguments.txt'))
# Store some git revision info in a text file in the log directory
src_path, _ = os.path.split(os.path.realpath(__file__))
utils.store_revision_info(src_path, log_dir, ' '.join(sys.argv))
np.random.seed(seed=args.seed)
train_set = utils.get_dataset(args.data_dir)
#train_set = facenet.dataset_from_list2(args.data_dir,'dataset/casia_maxpy_mtcnnpy_182',error_classes=[],drop_key='AsianStarCropBig_YES')
nrof_classes = len(train_set)
print('nrof_classes: ', nrof_classes)
image_list, label_list = utils.get_image_paths_and_labels(train_set)
image_list = np.array(image_list)
label_list = np.array(label_list, dtype=np.int32)
dataset_size = len(image_list)
single_batch_size = args.people_per_batch * args.images_per_person
indices = range(dataset_size)
np.random.shuffle(indices)
def _sample_people_softmax(x):
global softmax_ind
if softmax_ind >= dataset_size:
np.random.shuffle(indices)
softmax_ind = 0
true_num_batch = min(single_batch_size, dataset_size - softmax_ind)
sample_paths = image_list[indices[softmax_ind:softmax_ind +
true_num_batch]]
sample_labels = label_list[indices[softmax_ind:softmax_ind +
true_num_batch]]
softmax_ind += true_num_batch
return (np.array(sample_paths), np.array(sample_labels,
dtype=np.int32))
def _sample_people(x):
'''We sample people based on tf.data, where we can use transform and prefetch.
'''
image_paths, num_per_class = sample_people(
train_set, args.people_per_batch * (args.num_gpus - 1),
args.images_per_person)
labels = []
for i in range(len(num_per_class)):
labels.extend([i] * num_per_class[i])
return (np.array(image_paths), np.array(labels, dtype=np.int32))
def _parse_function(filename, label):
file_contents = tf.read_file(filename)
image = tf.image.decode_image(file_contents, channels=3)
#image = tf.image.decode_jpeg(file_contents, channels=3)
print(image.shape)
if args.random_crop:
print('use random crop')
image = tf.random_crop(image,
[args.image_size, args.image_size, 3])
else:
print('Not use random crop')
#image.set_shape((args.image_size, args.image_size, 3))
image.set_shape((None, None, 3))
image = tf.image.resize_images(image,
size=(args.image_height,
args.image_width))
#print(image.shape)
if args.random_flip:
image = tf.image.random_flip_left_right(image)
#pylint: disable=no-member
#image.set_shape((args.image_size, args.image_size, 3))
image.set_shape((args.image_height, args.image_width, 3))
if debug:
image = tf.cast(image, tf.float32)
else:
image = tf.image.per_image_standardization(image)
return image, label
#train_set = facenet.dataset_from_list(args.data_dir,'dataset/ms_mp',keys=['MultiPics'])
#train_set = facenet.dataset_from_list(args.data_dir,'dataset/ms_mp')
gpus = [0, 1]
#gpus = [0]
print('Model directory: %s' % model_dir)
print('Log directory: %s' % log_dir)
if args.pretrained_model:
print('Pre-trained model: %s' %
os.path.expanduser(args.pretrained_model))
with tf.Graph().as_default():
tf.set_random_seed(args.seed)
global_step = tf.Variable(0, trainable=False, name='global_step')
# Placeholder for the learning rate
learning_rate_placeholder = tf.placeholder(tf.float32,
name='learning_rate')
phase_train_placeholder = tf.placeholder(tf.bool, name='phase_train')
#the image is generated by sequence
with tf.device("/cpu:0"):
softmax_dataset = tf_data.Dataset.range(args.epoch_size *
args.max_nrof_epochs * 100)
softmax_dataset = softmax_dataset.map(lambda x: tf.py_func(
_sample_people_softmax, [x], [tf.string, tf.int32]))
softmax_dataset = softmax_dataset.flat_map(_from_tensor_slices)
softmax_dataset = softmax_dataset.map(_parse_function,
num_threads=8,
output_buffer_size=2000)
softmax_dataset = softmax_dataset.batch(args.num_gpus *
single_batch_size)
softmax_iterator = softmax_dataset.make_initializable_iterator()
softmax_next_element = softmax_iterator.get_next()
softmax_next_element[0].set_shape(
(args.num_gpus * single_batch_size, args.image_height,
args.image_width, 3))
softmax_next_element[1].set_shape(args.num_gpus *
single_batch_size)
batch_image_split = tf.split(softmax_next_element[0],
args.num_gpus)
batch_label_split = tf.split(softmax_next_element[1],
args.num_gpus)
learning_rate = tf.train.exponential_decay(
learning_rate_placeholder,
global_step,
args.learning_rate_decay_epochs * args.epoch_size,
args.learning_rate_decay_factor,
staircase=True)
tf.summary.scalar('learning_rate', learning_rate)
print('Using optimizer: {}'.format(args.optimizer))
if args.optimizer == 'ADAGRAD':
opt = tf.train.AdagradOptimizer(learning_rate)
elif args.optimizer == 'MOM':
opt = tf.train.MomentumOptimizer(learning_rate, 0.9)
tower_losses = []
tower_cross = []
tower_dist = []
tower_th = []
for i in range(args.num_gpus):
with tf.device("/gpu:" + str(i)):
with tf.name_scope("tower_" + str(i)) as scope:
with slim.arg_scope([slim.model_variable, slim.variable],
device="/cpu:0"):
with tf.variable_scope(
tf.get_variable_scope()) as var_scope:
reuse = False if i == 0 else True
#with slim.arg_scope(resnet_v2.resnet_arg_scope(args.weight_decay)):
#prelogits, end_points = resnet_v2.resnet_v2_50(batch_image_split[i],is_training=True,
# output_stride=16,num_classes=args.embedding_size,reuse=reuse)
#prelogits, end_points = network.inference(batch_image_split[i], args.keep_probability,
# phase_train=phase_train_placeholder, bottleneck_layer_size=args.embedding_size,
# weight_decay=args.weight_decay, reuse=reuse)
if args.network == 'slim_sphere':
prelogits = network.infer(batch_image_split[i])
elif args.network == 'densenet':
with slim.arg_scope(
densenet.densenet_arg_scope(
args.weight_decay)):
#prelogits, endpoints = densenet.densenet_small(batch_image_split[i],num_classes=args.embedding_size,is_training=True,reuse=reuse)
prelogits, endpoints = densenet.densenet_small_middle(
batch_image_split[i],
num_classes=args.embedding_size,
is_training=True,
reuse=reuse)
prelogits = tf.squeeze(prelogits,
axis=[1, 2])
#prelogits = slim.batch_norm(prelogits, is_training=True, decay=0.997,epsilon=1e-5,scale=True,updates_collections=tf.GraphKeys.UPDATE_OPS,reuse=reuse,scope='softmax_bn')
if args.loss_type == 'softmax':
cross_entropy_mean = utils.softmax_loss(
prelogits, batch_label_split[i],
len(train_set), args.weight_decay, reuse)
regularization_losses = tf.get_collection(
tf.GraphKeys.REGULARIZATION_LOSSES)
tower_cross.append(cross_entropy_mean)
#loss = cross_entropy_mean + args.weight_decay*tf.add_n(regularization_losses)
loss = cross_entropy_mean + tf.add_n(
regularization_losses)
tower_dist.append(0)
tower_cross.append(cross_entropy_mean)
tower_th.append(0)
tower_losses.append(loss)
elif args.loss_type == 'scatter' or args.loss_type == 'coco':
label_reshape = tf.reshape(
batch_label_split[i], [single_batch_size])
label_reshape = tf.cast(
label_reshape, tf.int64)
if args.loss_type == 'scatter':
scatter_loss, _ = utils.weight_scatter_speed(
prelogits,
label_reshape,
len(train_set),
reuse,
weight=args.weight,
scale=args.scale)
else:
scatter_loss, _ = utils.coco_loss(
prelogits,
label_reshape,
len(train_set),
reuse,
alpha=args.alpha,
scale=args.scale)
regularization_losses = tf.get_collection(
tf.GraphKeys.REGULARIZATION_LOSSES)
loss = scatter_loss[
'loss_total'] + args.weight_decay * tf.add_n(
regularization_losses)
tower_dist.append(scatter_loss['loss_dist'])
tower_cross.append(0)
tower_th.append(scatter_loss['loss_th'])
tower_losses.append(loss)
#loss = tf.add_n([cross_entropy_mean] + regularization_losses, name='total_loss')
tf.get_variable_scope().reuse_variables()
total_loss = tf.reduce_mean(tower_losses)
total_cross = tf.reduce_mean(tower_cross)
total_dist = tf.reduce_mean(tower_dist)
total_th = tf.reduce_mean(tower_th)
losses = {}
losses['total_loss'] = total_loss
losses['total_cross'] = total_cross
losses['total_dist'] = total_dist
losses['total_th'] = total_th
debug_info = {}
debug_info['logits'] = prelogits
#debug_info['end_points'] = end_points
debug_info['batch_image_split'] = batch_image_split
debug_info['batch_label_split'] = batch_label_split
#debug_info['endpoints'] = endpoints
grads = opt.compute_gradients(total_loss,
tf.trainable_variables(),
colocate_gradients_with_ops=True)
apply_gradient_op = opt.apply_gradients(grads, global_step=global_step)
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
train_op = tf.group(apply_gradient_op)
save_vars = [
var for var in tf.global_variables()
if 'Adagrad' not in var.name and 'global_step' not in var.name
]
check_nan = tf.add_check_numerics_ops()
debug_info['check_nan'] = check_nan
#saver = tf.train.Saver(tf.trainable_variables(), max_to_keep=3)
saver = tf.train.Saver(save_vars, max_to_keep=3)
# Build the summary operation based on the TF collection of Summaries.
summary_op = tf.summary.merge_all()
# Start running operations on the Graph.
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=args.gpu_memory_fraction)
sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options,
allow_soft_placement=True))
# Initialize variables
sess.run(tf.global_variables_initializer(),
feed_dict={phase_train_placeholder: True})
sess.run(tf.local_variables_initializer(),
feed_dict={phase_train_placeholder: True})
#sess.run(iterator.initializer)
sess.run(softmax_iterator.initializer)
summary_writer = tf.summary.FileWriter(log_dir, sess.graph)
coord = tf.train.Coordinator()
tf.train.start_queue_runners(coord=coord, sess=sess)
with sess.as_default():
#pdb.set_trace()
if args.pretrained_model:
print('Restoring pretrained model: %s' % args.pretrained_model)
saver.restore(sess, os.path.expanduser(args.pretrained_model))
# Training and validation loop
epoch = 0
while epoch < args.max_nrof_epochs:
step = sess.run(global_step, feed_dict=None)
epoch = step // args.epoch_size
if debug:
debug_train(args, sess, train_set, epoch,
image_batch_gather, enqueue_op,
batch_size_placeholder, image_batch_split,
image_paths_split, num_per_class_split,
image_paths_placeholder,
image_paths_split_placeholder,
labels_placeholder, labels_batch,
num_per_class_placeholder,
num_per_class_split_placeholder, len(gpus))
# Train for one epoch
train(args, sess, epoch, len(gpus), debug_info,
learning_rate_placeholder, phase_train_placeholder,
global_step, losses, train_op, summary_op,
summary_writer, args.learning_rate_schedule_file)
# Save variables and the metagraph if it doesn't exist already
save_variables_and_metagraph(sess, saver, summary_writer,
model_dir, subdir, step)
# Evaluate on LFW
return model_dir
def main(args):
#network = importlib.import_module(args.model_def)
subdir = datetime.strftime(datetime.now(), '%Y%m%d-%H%M%S')
log_dir = os.path.join(os.path.expanduser(args.logs_base_dir), subdir)
if not os.path.isdir(
log_dir): # Create the log directory if it doesn't exist
os.makedirs(log_dir)
model_dir = os.path.join(os.path.expanduser(args.models_base_dir), subdir)
if not os.path.isdir(
model_dir): # Create the model directory if it doesn't exist
os.makedirs(model_dir)
# Write arguments to a text file
utils.write_arguments_to_file(args, os.path.join(log_dir, 'arguments.txt'))
# Store some git revision info in a text file in the log directory
src_path, _ = os.path.split(os.path.realpath(__file__))
utils.store_revision_info(src_path, log_dir, ' '.join(sys.argv))
np.random.seed(seed=args.seed)
train_set = utils.get_dataset(args.data_dir)
nrof_classes = len(train_set)
print('nrof_classes: ', nrof_classes)
image_list, label_list = utils.get_image_paths_and_labels(train_set)
image_list = np.array(image_list)
print('total images: {}'.format(len(image_list)))
label_list = np.array(label_list, dtype=np.int32)
dataset_size = len(image_list)
data_reader = DataGenerator(image_list, label_list, args.batch_size)
print('Model directory: %s' % model_dir)
print('Log directory: %s' % log_dir)
if args.pretrained_model:
print('Pre-trained model: %s' %
os.path.expanduser(args.pretrained_model))
with tf.Graph().as_default():
tf.set_random_seed(args.seed)
global_step = tf.Variable(0, trainable=False, name='global_step')
# Placeholder for the learning rate
learning_rate_placeholder = tf.placeholder(tf.float32,
name='learning_rate')
images_placeholder = tf.placeholder(tf.float32, [None, 112, 96, 3],
name='images_placeholder')
labels_placeholder = tf.placeholder(tf.int32, [None],
name='labels_placeholder')
phase_train_placeholder = tf.placeholder(tf.bool, name='phase_train')
learning_rate = tf.train.exponential_decay(
learning_rate_placeholder,
global_step,
args.learning_rate_decay_epochs * args.epoch_size,
args.learning_rate_decay_factor,
staircase=True)
tf.summary.scalar('learning_rate', learning_rate)
print('Using optimizer: {}'.format(args.optimizer))
if args.optimizer == 'ADAGRAD':
opt = tf.train.AdagradOptimizer(learning_rate)
elif args.optimizer == 'MOM':
opt = tf.train.MomentumOptimizer(learning_rate, 0.9)
if args.network == 'sphere_network':
prelogits = network.infer(images_placeholder)
else:
raise Exception('Not supported network: {}'.format(args.loss_type))
if args.loss_type == 'softmax':
cross_entropy_mean = utils.softmax_loss(prelogits,
labels_placeholder,
len(train_set),
args.weight_decay, False)
regularization_losses = tf.get_collection(
tf.GraphKeys.REGULARIZATION_LOSSES)
#loss = cross_entropy_mean + args.weight_decay*tf.add_n(regularization_losses)
loss = cross_entropy_mean + args.weight_decay * tf.add_n(
regularization_losses)
#loss = cross_entropy_mean
else:
raise Exception('Not supported loss type: {}'.format(
args.loss_type))
#loss = tf.add_n([cross_entropy_mean] + regularization_losses, name='total_loss')
losses = {}
losses['total_loss'] = loss
losses['softmax_loss'] = cross_entropy_mean
debug_info = {}
debug_info['prelogits'] = prelogits
grads = opt.compute_gradients(loss, tf.trainable_variables())
train_op = opt.apply_gradients(grads, global_step=global_step)
#save_vars = [var for var in tf.global_variables() if 'Adagrad' not in var.name and 'global_step' not in var.name]
save_vars = tf.global_variables()
#saver = tf.train.Saver(tf.trainable_variables(), max_to_keep=3)
saver = tf.train.Saver(save_vars, max_to_keep=3)
# Build the summary operation based on the TF collection of Summaries.
# Start running operations on the Graph.
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=args.gpu_memory_fraction)
sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options,
allow_soft_placement=True))
# Initialize variables
sess.run(tf.global_variables_initializer(),
feed_dict={phase_train_placeholder: True})
sess.run(tf.local_variables_initializer(),
feed_dict={phase_train_placeholder: True})
with sess.as_default():
#pdb.set_trace()
if args.pretrained_model:
print('Restoring pretrained model: %s' % args.pretrained_model)
saver.restore(sess, os.path.expanduser(args.pretrained_model))
# Training and validation loop
epoch = 0
while epoch < args.max_nrof_epochs:
step = sess.run(global_step, feed_dict=None)
epoch = step // args.epoch_size
# Train for one epoch
train(args, sess, epoch, images_placeholder,
labels_placeholder, data_reader, debug,
learning_rate_placeholder, global_step, losses, train_op,
args.learning_rate_schedule_file)
# Save variables and the metagraph if it doesn't exist already
model_dir = args.models_base_dir
checkpoint_path = os.path.join(model_dir,
'model-%s.ckpt' % 'softmax')
saver.save(sess,
checkpoint_path,
global_step=step,
write_meta_graph=False)
# Evaluate on LFW
return model_dir
def main_train(args):
subdir = datetime.strftime(datetime.now(), '%Y%m%d-%H%M%S')
log_dir = os.path.join(os.path.expanduser(args.logs_base_dir), subdir)
if not os.path.isdir(
log_dir): # Create the log directory if it doesn't exist
os.makedirs(log_dir)
model_dir = os.path.join(os.path.expanduser(args.models_base_dir), subdir)
if not os.path.isdir(
model_dir): # Create the model directory if it doesn't exist
os.makedirs(model_dir)
# Write arguments to a text file
utils.write_arguments_to_file(args, os.path.join(log_dir, 'arguments.txt'))
# Store some git revision info in a text file in the log directory
src_path, _ = os.path.split(os.path.realpath(__file__))
utils.store_revision_info(src_path, log_dir, ' '.join(sys.argv))
np.random.seed(seed=args.seed)
train_set = utils.dataset_from_list(
args.train_data_dir, args.train_list_dir) # class objects in a list
#----------------------class definition-------------------------------------
'''
class ImageClass():
"Stores the paths to images for a given class"
def __init__(self, name, image_paths):
self.name = name
self.image_paths = image_paths
def __str__(self):
return self.name + ', ' + str(len(self.image_paths)) + ' images'
def __len__(self):
return len(self.image_paths)
'''
nrof_classes = len(train_set)
print('nrof_classes: ', nrof_classes)
image_list, label_list = utils.get_image_paths_and_labels(train_set)
print('total images: ', len(image_list)) # label is in the form scalar.
image_list = np.array(image_list)
label_list = np.array(label_list, dtype=np.int32)
dataset_size = len(image_list)
single_batch_size = args.class_per_batch * args.images_per_class
indices = list(range(dataset_size))
np.random.shuffle(indices)
def _sample_people_softmax(x): # loading the images in batches.
global softmax_ind
if softmax_ind >= dataset_size:
np.random.shuffle(indices)
softmax_ind = 0
true_num_batch = min(single_batch_size, dataset_size - softmax_ind)
sample_paths = image_list[indices[softmax_ind:softmax_ind +
true_num_batch]]
sample_images = []
for item in sample_paths:
sample_images.append(np.load(str(item)))
#print(item)
#print(type(sample_paths[0]))
sample_labels = label_list[indices[softmax_ind:softmax_ind +
true_num_batch]]
softmax_ind += true_num_batch
return (np.expand_dims(np.array(sample_images, dtype=np.float32),
axis=4), np.array(sample_labels,
dtype=np.int32))
print('Model directory: %s' % model_dir)
print('Log directory: %s' % log_dir)
if args.pretrained_model:
print('Pre-trained model: %s' %
os.path.expanduser(args.pretrained_model))
with tf.Graph().as_default():
tf.set_random_seed(args.seed)
global_step = tf.Variable(0, trainable=False, name='global_step')
# Placeholder for the learning rate
learning_rate_placeholder = tf.placeholder(tf.float32,
name='learning_rate')
phase_train_placeholder = tf.placeholder(tf.bool, name='phase_train')
#the image is generated by sequence
with tf.device("/cpu:0"):
softmax_dataset = tf.data.Dataset.range(args.epoch_size *
args.max_nrof_epochs)
softmax_dataset = softmax_dataset.map(lambda x: tf.py_func(
_sample_people_softmax, [x], [tf.float32, tf.int32]))
softmax_dataset = softmax_dataset.flat_map(_from_tensor_slices)
softmax_dataset = softmax_dataset.batch(single_batch_size)
softmax_iterator = softmax_dataset.make_initializable_iterator()
softmax_next_element = softmax_iterator.get_next()
softmax_next_element[0].set_shape(
(single_batch_size, args.image_height, args.image_width,
args.image_width, 1))
softmax_next_element[1].set_shape(single_batch_size)
batch_image_split = softmax_next_element[0]
# batch_image_split = tf.expand_dims(batch_image_split, axis = 4)
batch_label_split = softmax_next_element[1]
learning_rate = tf.train.exponential_decay(
learning_rate_placeholder,
global_step,
args.learning_rate_decay_epochs * args.epoch_size,
args.learning_rate_decay_factor,
staircase=True)
tf.summary.scalar('learning_rate', learning_rate)
print('Using optimizer: {}'.format(args.optimizer))
if args.optimizer == 'ADAGRAD':
opt = tf.train.AdagradOptimizer(learning_rate)
elif args.optimizer == 'SGD':
opt = tf.train.GradientDescentOptimizer(learning_rate)
elif args.optimizer == 'MOM':
opt = tf.train.MomentumOptimizer(learning_rate, 0.9)
elif args.optimizer == 'ADAM':
opt = tf.train.AdamOptimizer(learning_rate,
beta1=0.9,
beta2=0.999,
epsilon=0.1)
else:
raise Exception("Not supported optimizer: {}".format(
args.optimizer))
losses = {}
with slim.arg_scope([slim.model_variable, slim.variable],
device="/cpu:0"):
with tf.variable_scope(tf.get_variable_scope()) as var_scope:
reuse = False
if args.network == 'sphere_network':
prelogits = network.infer(batch_image_split,
args.embedding_size)
else:
raise Exception("Not supported network: {}".format(
args.network))
if args.fc_bn:
prelogits = slim.batch_norm(prelogits, is_training=True, decay=0.997,epsilon=1e-5,scale=True,\
updates_collections=tf.GraphKeys.UPDATE_OPS,reuse=reuse,scope='softmax_bn')
if args.loss_type == 'softmax':
cross_entropy_mean = utils.softmax_loss(
prelogits, batch_label_split, len(train_set), 1.0,
reuse)
regularization_losses = tf.get_collection(
tf.GraphKeys.REGULARIZATION_LOSSES)
loss = cross_entropy_mean + args.weight_decay * tf.add_n(
regularization_losses)
print('************************' +
' Computing the softmax loss')
losses['total_loss'] = cross_entropy_mean
losses['total_reg'] = args.weight_decay * tf.add_n(
regularization_losses)
elif args.loss_type == 'lmcl':
label_reshape = tf.reshape(batch_label_split,
[single_batch_size])
label_reshape = tf.cast(label_reshape, tf.int64)
coco_loss = utils.cos_loss(prelogits,
label_reshape,
len(train_set),
reuse,
alpha=args.alpha,
scale=args.scale)
regularization_losses = tf.get_collection(
tf.GraphKeys.REGULARIZATION_LOSSES)
loss = coco_loss + args.weight_decay * tf.add_n(
regularization_losses)
print('************************' +
' Computing the lmcl loss')
losses['total_loss'] = coco_loss
losses['total_reg'] = args.weight_decay * tf.add_n(
regularization_losses)
elif args.loss_type == 'center':
# center loss
center_loss, centers, centers_update_op = get_center_loss(prelogits, label_reshape, args.center_loss_alfa, \
args.num_class_train)
regularization_losses = tf.get_collection(
tf.GraphKeys.REGULARIZATION_LOSSES)
loss = center_loss + args.weight_decay * tf.add_n(
regularization_losses)
print('************************' +
' Computing the center loss')
losses['total_loss'] = center_loss
losses['total_reg'] = args.weight_decay * tf.add_n(
regularization_losses)
elif args.loss_type == 'lmccl':
cross_entropy_mean = utils.softmax_loss(
prelogits, batch_label_split, len(train_set), 1.0,
reuse)
label_reshape = tf.reshape(batch_label_split,
[single_batch_size])
label_reshape = tf.cast(label_reshape, tf.int64)
coco_loss = utils.cos_loss(prelogits,
label_reshape,
len(train_set),
reuse,
alpha=args.alpha,
scale=args.scale)
center_loss, centers, centers_update_op = get_center_loss(prelogits, label_reshape, args.center_loss_alfa, \
args.num_class_train)
regularization_losses = tf.get_collection(
tf.GraphKeys.REGULARIZATION_LOSSES)
reg_loss = args.weight_decay * tf.add_n(
regularization_losses)
loss = coco_loss + reg_loss + args.center_weighting * center_loss + cross_entropy_mean
losses[
'total_loss_center'] = args.center_weighting * center_loss
losses['total_loss_lmcl'] = coco_loss
losses['total_loss_softmax'] = cross_entropy_mean
losses['total_reg'] = reg_loss
grads = opt.compute_gradients(loss,
tf.trainable_variables(),
colocate_gradients_with_ops=True)
apply_gradient_op = opt.apply_gradients(grads, global_step=global_step)
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
# used for updating the centers in the center loss.
if args.loss_type == 'lmccl' or args.loss_type == 'center':
with tf.control_dependencies([centers_update_op]):
with tf.control_dependencies(update_ops):
train_op = tf.group(apply_gradient_op)
else:
with tf.control_dependencies(update_ops):
train_op = tf.group(apply_gradient_op)
save_vars = [
var for var in tf.global_variables()
if 'Adagrad' not in var.name and 'global_step' not in var.name
]
saver = tf.train.Saver(save_vars, max_to_keep=3)
# Build the summary operation based on the TF collection of Summaries.
summary_op = tf.summary.merge_all()
# Start running operations on the Graph.
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=args.gpu_memory_fraction)
sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options,
allow_soft_placement=True))
# Initialize variables
sess.run(tf.global_variables_initializer(),
feed_dict={phase_train_placeholder: True})
sess.run(tf.local_variables_initializer(),
feed_dict={phase_train_placeholder: True})
#sess.run(iterator.initializer)
sess.run(softmax_iterator.initializer)
summary_writer = tf.summary.FileWriter(log_dir, sess.graph)
coord = tf.train.Coordinator()
tf.train.start_queue_runners(coord=coord, sess=sess)
with sess.as_default():
if args.pretrained_model:
print('Restoring pretrained model: %s' % args.pretrained_model)
saver.restore(sess, os.path.expanduser(args.pretrained_model))
# Training and validation loop
epoch = 0
while epoch < args.max_nrof_epochs:
step = sess.run(global_step, feed_dict=None)
epoch = step // args.epoch_size
if debug:
debug_train(args, sess, train_set, epoch, image_batch_gather,\
enqueue_op,batch_size_placeholder, image_batch_split,image_paths_split,num_per_class_split,
image_paths_placeholder,image_paths_split_placeholder, labels_placeholder, labels_batch,\
num_per_class_placeholder,num_per_class_split_placeholder,len(gpus))
# Train for one epoch
if args.loss_type == 'lmccl' or args.loss_type == 'center':
train_contain_center(args, sess, epoch,
learning_rate_placeholder,
phase_train_placeholder, global_step,
losses, train_op, summary_op,
summary_writer, '', centers_update_op)
else:
train(args, sess, epoch, learning_rate_placeholder,
phase_train_placeholder, global_step, losses,
train_op, summary_op, summary_writer, '')
# Save variables and the metagraph if it doesn't exist already
save_variables_and_metagraph(sess, saver, summary_writer,
model_dir, subdir, step)
return model_dir
def main(argv=None):
if tf.gfile.Exists(FLAGS.train_dir):
tf.gfile.DeleteRecursively(FLAGS.train_dir)
tf.gfile.MakeDirs(FLAGS.train_dir)
global_step = tf.contrib.framework.get_or_create_global_step()
save_path = os.path.join(FLAGS.train_dir, 'model_ckpt')
#获取(image, label)batch pair
image_batch, label_batch = inputs(data_type='train')
#损失函数sparse_softmax_cross_entropy_with_logits要求rank_of_labels = rank_of_images - 1
#对label_batch作扁平化处理
label_batch = tf.reshape(label_batch, [50])
#扩展image维度，从[batch, row, col]转换为[batch, row, col, depth=1]
expand_image_batch = tf.expand_dims(image_batch, -1)
input_placeholder = tf.placeholder_with_default(expand_image_batch,
shape=[None, 28, 28, 1],
name='input')
# 构建模型
# 第一个卷积层
with tf.variable_scope('conv1') as scope:
kernal = weight_variable('weights', shape=[5, 5, 1, 32])
biases = bias_variable('biases', shape=[32])
pre_activation = tf.nn.bias_add(conv2d(input_placeholder, kernal),
biases)
conv1 = tf.nn.relu(pre_activation, name=scope.name)
# 第一个池化层
pool1 = max_pool(conv1)
# 第二个卷积层
with tf.variable_scope('conv2') as scope:
kernal = weight_variable('weights', shape=[5, 5, 32, 64])
biases = bias_variable('biases', shape=[64])
pre_activation = tf.nn.bias_add(conv2d(pool1, kernal), biases)
conv2 = tf.nn.relu(pre_activation, name=scope.name)
# 第二个池化层
# 7*7*64
pool2 = max_pool(conv2)
# 全连接层
with tf.variable_scope('fc1') as scope:
weight_fc1 = weight_variable('weights', shape=[7 * 7 * 64, 1024])
biases = bias_variable('biases', shape=[1024])
pool2_flat = tf.reshape(pool2, [-1, 7 * 7 * 64])
fc1 = tf.nn.relu((tf.matmul(pool2_flat, weight_fc1) + biases),
name=scope.name)
print('Tensor fc1/relu: ', fc1.name)
keep_prob = tf.placeholder(tf.float32, name='keep_prob')
fc1_drop = tf.nn.dropout(fc1, keep_prob, name='fc1_drop')
print('>>Tensor dropout: ', fc1_drop.name)
# 输出层
with tf.variable_scope('softmax_linear') as scope:
weight_fc2 = weight_variable('weight', shape=[1024, 10])
biases = bias_variable('biases', shape=[10])
softmax_output = tf.add(tf.matmul(fc1_drop, weight_fc2),
biases,
name=scope.name)
print('>>Tensor softmax_linear/softmax_output: ', softmax_output.name)
loss = softmax_loss(logits=softmax_output, labels=label_batch)
print('>>Tensor loss: ', loss.name)
accuracy = train_accuracy(softmax_output, label_batch)
print('>>Tensor accuracy: ', accuracy.name)
train_op = train(loss, global_step)
print('>>Tensor train_op: ', train_op.name)
#初始化所有参数
init = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init)
coord = tf.train.Coordinator()
try:
threads = []
for qr in tf.get_collection(tf.GraphKeys.QUEUE_RUNNERS):
threads.extend(
qr.create_threads(sess, coord, daemon=True, start=True))
saver = tf.train.Saver()
step = 1
while step <= 20000 and not coord.should_stop():
if step % 100 == 0:
# 每隔100步打印一次accuracy
runtime_accuracy = sess.run(accuracy,
feed_dict={keep_prob: 1.0})
print(">>step %d, training accuracy %g" %
(step, runtime_accuracy))
# 每隔1000步保存一次模型
if step % 1000 == 0:
saver.save(sess, save_path, global_step=step)
# 训练模型
sess.run(train_op, feed_dict={keep_prob: 0.5})
# 步数更新
step += 1
except Exception as e:
coord.request_stop(e)
coord.request_stop()
coord.join(threads, stop_grace_period_secs=10)
