def build_loss(logits, labels, global_step, config):
if hasattr(tf.nn, "softmax_cross_entropy_with_logits_v2"):
loss_op = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(
labels=labels, logits=logits),
name="softmax_cross_entropy")
else:
loss_op = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(
labels=labels, logits=logits),
name="softmax_cross_entropy")
# todo : tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)
# loss = tf.add(loss, tf.reduce_sum(tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)))
if config.use_regularizer:
weights = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)
regularizer = 0
for weight in weights:
regularizer += tf.nn.l2_loss(weight)
regularizer *= config.weight_decay
loss_op += regularizer
learning_rate = optimizer.configure_learning_rate(global_step, config)
opt = optimizer.configure_optimizer(learning_rate, config)
train_op = opt.minimize(loss_op, global_step=global_step)
accuracy_op = tf.reduce_mean(
tf.cast(tf.equal(tf.argmax(logits, 1), tf.argmax(labels, 1)),
tf.float32))
pred_idx_op = tf.argmax(logits, 1)
return {
"loss": loss_op,
"train": train_op,
"accuracy": accuracy_op,
"learning_rate": learning_rate,
"pred_idx": pred_idx_op
}
def train_op_fun(total_loss, global_step, cf):
"""Train model.
Create an optimizer and apply to all trainable variables. Add moving
average for all trainable variables.
Args:
total_loss: Total loss from loss().
global_step: Integer Variable counting the number of training steps
processed.
Returns:
train_op: op for training.
"""
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
lr = optimizer.configure_learning_rate(global_step, cf)
tf.summary.scalar('learning_rate', lr)
opt = optimizer.configure_optimizer(lr, cf)
variables_to_train = _get_variables_to_train(cf)
grads = opt.compute_gradients(total_loss, variables_to_train)
grad_updates = opt.apply_gradients(grads, global_step=global_step)
update_ops.append(grad_updates)
update_op = tf.group(*update_ops)
with tf.control_dependencies([update_op]):
train_op = tf.identity(total_loss, name='train_op')
return train_op, lr
def build_loss(logits, labels, global_step, class_weights_ph, config):
loss_op = tf.reduce_mean(logits + tf.log(0.1 + tf.exp(-logits)))
# if config.use_weighted_loss:
# logits_class_weights = tf.reduce_sum(labels * class_weights_ph, axis=1)
# loss_op = tf.losses.softmax_cross_entropy(labels, logits, weights=logits_class_weights)
# else:
# if hasattr(tf.nn, "softmax_cross_entropy_with_logits_v2"):
# loss_op = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(labels=labels, logits=logits),
# name="softmax_cross_entropy")
# else:
# loss_op = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=labels, logits=logits),
# name="softmax_cross_entropy")
# # todo : tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)
# # loss = tf.add(loss, tf.reduce_sum(tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)))
result = {"loss": loss_op}
if config.train:
learning_rate = optimizer.configure_learning_rate(global_step, config)
opt = optimizer.configure_optimizer(learning_rate, config)
train_op = opt.minimize(loss_op, global_step=global_step)
result["learning_rate"] = learning_rate
result["train"] = train_op
accuracy_op = tf.reduce_mean(
tf.cast(tf.equal(tf.argmax(logits, 1), tf.argmax(labels, 1)),
tf.float32))
pred_idx_op = tf.argmax(logits, 1)
result["accuracy"] = accuracy_op
result["pred_idx"] = pred_idx_op
return result
def build_loss(logits_list, labels, global_step, class_weights_ph, config):
loss_op = None
probs = None
for logits in logits_list:
if hasattr(tf.nn, "softmax_cross_entropy_with_logits_v2"):
loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits_v2(logits=logits,
labels=labels))
else:
loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=logits,
labels=labels))
if loss_op is None:
loss_op = loss
else:
loss_op += loss
prob = tf.nn.softmax(logits)
if probs is None:
probs = prob
else:
probs += prob
# todo : tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)
# loss = tf.add(loss, tf.reduce_sum(tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)))
if config.use_regularizer:
weights = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)
regularizer = 0
for weight in weights:
regularizer += tf.nn.l2_loss(weight)
regularizer *= config.weight_decay
loss_op += regularizer
result = {"loss": loss_op}
loss_op /= len(logits_list)
if config.train:
learning_rate = optimizer.configure_learning_rate(global_step, config)
opt = optimizer.configure_optimizer(learning_rate, config)
train_op = opt.minimize(loss_op, global_step=global_step)
result["learning_rate"] = learning_rate
result["train"] = train_op
accuracy_op = tf.reduce_mean(
tf.cast(tf.equal(tf.argmax(probs, 1), tf.argmax(labels, 1)),
tf.float32))
pred_idx_op = tf.argmax(probs, 1)
result["accuracy"] = accuracy_op
result["pred_idx"] = pred_idx_op
return result
def build_model_multiple(config, dataset, model_f, is_training):
# labels = tf.placeholder(tf.float32, shape=[None, config.num_class], name="labels")
global_step = tf.Variable(0, trainable=False)
default_last_conv_name = None
learning_rate = optimizer.configure_learning_rate(global_step, config)
opt = optimizer.configure_optimizer(learning_rate, config)
tower_grads = []
if config.gpu_list:
gpu_list = [int(i) for i in config.gpu_list.split(",")]
else:
gpu_list = range(config.num_gpus)
with tf.variable_scope(tf.get_variable_scope()):
for idx, i in enumerate(gpu_list):
with tf.device('/gpu:%d' % i):
with tf.name_scope('%s_%d' % ("tower", i)) as scope:
if config.model_name in model_factory.networks_map.keys():
if config.model_name[:6] == "nasnet":
is_training = config.train
if hasattr(model_f, 'default_last_conv_name'):
default_last_conv_name = model_f.default_last_conv_name
# inputs = tf.placeholder(tf.float32, shape=[None, config.input_size, config.input_size, config.num_channel],
# name="inputs")
# tf.summary.image('input', inputs, config.num_input_summary)
logits, end_points = model_f(dataset.images[idx])
if config.model_name[:6] == "resnet":
logits = tf.reshape(logits, [-1, config.num_class])
else:
build_model_f = util.get_attr('model.%s' % config.model_name, "build_model")
if not build_model_f:
return None
if not config.input_size:
config.input_size = util.get_attr('model.%s' % config.model_name, "input_size")
if not config.input_size:
config.input_size = 224
default_last_conv_name = util.get_attr('model.%s' % config.model_name, "default_last_conv_name")
# inputs = tf.placeholder(tf.float32, shape=[None, config.input_size, config.input_size, config.num_channel],
# name="inputs")
tf.summary.image('input', dataset.images[idx], config.num_input_summary)
model_result = build_model_f(dataset.images[idx], config, is_training=is_training)
if isinstance(model_result, tuple):
logits = model_result[0]
end_points = model_result[1]
else:
logits = model_result
end_points = None
if hasattr(config, 'loss_file'):
loss_file = config.loss_file
else:
loss_file = "softmax_cross_entropy"
loss_f = util.get_attr('model.loss.%s' % loss_file, "build_loss_multiple")
ops = None
if loss_f:
loss = loss_f(logits, dataset.labels[idx], global_step, config, scope, opt)
tower_grads.append(loss)
grads = average_gradients(tower_grads)
apply_gradient_op = opt.apply_gradients(grads, global_step=global_step)
variable_averages = tf.train.ExponentialMovingAverage(config.moving_average_decay, global_step)
variables_averages_op = variable_averages.apply(tf.trainable_variables())
# Group all updates to into a single train op.
train_op = tf.group(apply_gradient_op, variables_averages_op)
return train_op, is_training, global_step, default_last_conv_name, tower_grads
def build_model(images, labels, mode, params):
is_training = (mode == tf.estimator.ModeKeys.TRAIN)
# -----------------------------------------------------------
# MODEL: define the layers of the model
with tf.variable_scope('model'):
# Compute the embeddings with the model
if params.model_name.startswith("slim."):
embeddings = build_slim_model(is_training, images, params)
else:
embeddings = build_custom_model(is_training, images, params)
embedding_mean_norm = tf.reduce_mean(tf.norm(embeddings, axis=1))
tf.summary.scalar("embedding_mean_norm", embedding_mean_norm)
if mode == tf.estimator.ModeKeys.PREDICT:
return embeddings
labels = tf.cast(labels, tf.int64)
# Define triplet loss
if params.triplet_strategy == "batch_all":
loss, fraction = batch_all_triplet_loss(labels,
embeddings,
margin=params.margin,
squared=params.squared)
elif params.triplet_strategy == "batch_hard":
loss = batch_hard_triplet_loss(labels,
embeddings,
margin=params.margin,
squared=params.squared)
else:
raise ValueError("Triplet strategy not recognized: {}".format(
params.triplet_strategy))
# -----------------------------------------------------------
# METRICS AND SUMMARIES
# Metrics for evaluation using tf.metrics (average over whole dataset)
# TODO: some other metrics like rank-1 accuracy?
with tf.variable_scope("metrics"):
eval_metric_ops = {
"embedding_mean_norm": tf.metrics.mean(embedding_mean_norm)
}
if params.triplet_strategy == "batch_all":
eval_metric_ops['fraction_positive_triplets'] = tf.metrics.mean(
fraction)
# Summaries for training
tf.summary.scalar('loss', loss)
if params.triplet_strategy == "batch_all":
tf.summary.scalar('fraction_positive_triplets', fraction)
tf.summary.image('train_image', images, max_outputs=1)
# Define training step that minimizes the loss with the Adam optimizer
global_step = tf.train.get_global_step()
learning_rate = optimizer.configure_learning_rate(global_step, config)
opt = optimizer.configure_optimizer(learning_rate, config)
optimizer = tf.train.AdamOptimizer(params.learning_rate)
train_op = optimizer.minimize(loss, global_step=global_step)
return train_op, embeddings, loss