def model(model_name,
x_input,
num_classes,
preprocess=False,
is_training=False,
label_offset=0,
scope=None,
reuse=None):
network_fn = nets_factory.get_network_fn(model_name,
num_classes=num_classes -
label_offset,
is_training=is_training,
scope=scope,
reuse=reuse)
eval_image_size = network_fn.default_image_size
print('model[' + model_name + '] eval_image_size:', eval_image_size)
images = x_input
if preprocess:
# images = preprocess_batch(model_name, x_input, eval_image_size, eval_image_size)
if model_name.startswith('resnet_v1') or model_name.startswith('vgg'):
images = vgg_preprocess(x_input, eval_image_size, eval_image_size)
# if model_name.startswith('inception_v'):
# images = inception_preprocess(x_input, eval_image_size, eval_image_size)
logits, _ = network_fn(images)
return logits
def make_style_net(self, images):
with tf.device("/gpu:0"):
network_fn = nets_factory.get_network_fn('inception_resnet_v2',
num_classes=27,
is_training=False)
if images.shape[1:3] != (256, 256):
images = tf.image.resize_images(images, [256, 256])
logits, _ = network_fn(images)
logits = tf.stop_gradient(logits)
return logits
def testGetNetworkFnArgScope(self):
batch_size = 5
num_classes = 10
net = 'cifarnet'
with self.test_session(use_gpu=True):
net_fn = nets_factory.get_network_fn(net, num_classes)
image_size = getattr(net_fn, 'default_image_size', 224)
with slim.arg_scope([slim.model_variable, slim.variable],
device='/CPU:0'):
inputs = tf.random_uniform((batch_size, image_size, image_size, 3))
net_fn(inputs)
weights = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, 'CifarNet/conv1')[0]
self.assertDeviceEqual('/CPU:0', weights.device)
def testGetNetworkFnSecondHalf(self):
batch_size = 5
num_classes = 1000
for net in nets_factory.networks_map.keys()[10:]:
with tf.Graph().as_default() as g, self.test_session(g):
net_fn = nets_factory.get_network_fn(net, num_classes)
# Most networks use 224 as their default_image_size
image_size = getattr(net_fn, 'default_image_size', 224)
inputs = tf.random_uniform((batch_size, image_size, image_size, 3))
logits, end_points = net_fn(inputs)
self.assertTrue(isinstance(logits, tf.Tensor))
self.assertTrue(isinstance(end_points, dict))
self.assertEqual(logits.get_shape().as_list()[0], batch_size)
self.assertEqual(logits.get_shape().as_list()[-1], num_classes)
def inference_on_image(bot_id, suffix, setting_id, image_file, network_name='inception_v4', return_labels=1):
"""
Loads the corresponding model checkpoint, network function and preprocessing routine based on bot_id and network_name,
restores the graph and runs it to the prediction enpoint with the image as input
:param bot_id: bot_id, used to reference to correct model directory
:param image_file: reference to the temporary image file to be classified
:param network_name: name of the network type to be used
:param return_labels: number of labels to return
:return: the top n labels with probabilities, where n = return_labels
"""
# Get the model path
model_path = dirs.get_transfer_model_dir(bot_id+suffix, setting_id)
# Get number of classes to predict
protobuf_dir = dirs.get_transfer_proto_dir(bot_id, setting_id)
number_of_classes = dataset_utils.get_number_of_classes_by_labels(protobuf_dir)
# Get the preprocessing and network construction functions
preprocessing_fn = preprocessing_factory.get_preprocessing(network_name, is_training=False)
network_fn = network_factory.get_network_fn(network_name, number_of_classes)
# Process the temporary image file into a Tensor of shape [widht, height, channels]
image_tensor = tf.gfile.FastGFile(image_file, 'rb').read()
image_tensor = tf.image.decode_image(image_tensor, channels=0)
# Perform preprocessing and reshape into [network.default_width, network.default_height, channels]
network_default_size = network_fn.default_image_size
image_tensor = preprocessing_fn(image_tensor, network_default_size, network_default_size)
# Create an input batch of size one from the preprocessed image
input_batch = tf.reshape(image_tensor, [1, 299, 299, 3])
# Create the network up to the Predictions Endpoint
logits, endpoints = network_fn(input_batch)
# Create a Saver() object to restore the network from the last checkpoint
restorer = tf.train.Saver()
with tf.Session() as sess:
tf.global_variables_initializer().run()
# Restore the variables of the network from the last checkpoint and run the graph
restorer.restore(sess, tf.train.latest_checkpoint(model_path))
sess.run(endpoints)
# Get the numpy array of predictions out of the
predictions = endpoints['Predictions'].eval()[0]
return map_predictions_to_labels(protobuf_dir, predictions, return_labels)
def testGetNetworkFnVideoModels(self):
batch_size = 5
num_classes = 400
for net in ['i3d', 's3dg']:
with tf.Graph().as_default() as g, self.test_session(g):
net_fn = nets_factory.get_network_fn(net,
num_classes=num_classes)
# Most networks use 224 as their default_image_size
image_size = getattr(net_fn, 'default_image_size', 224) // 2
inputs = tf.random_uniform(
(batch_size, 10, image_size, image_size, 3))
logits, end_points = net_fn(inputs)
self.assertTrue(isinstance(logits, tf.Tensor))
self.assertTrue(isinstance(end_points, dict))
self.assertEqual(logits.get_shape().as_list()[0], batch_size)
self.assertEqual(logits.get_shape().as_list()[-1], num_classes)
def main(argv):
logging.set_verbosity(tf.logging.INFO)
assert FLAGS.feature_extractor_name == 'resnet_v2_152'
# Get the function to preprocess and build network for the image.
net_fn = nets_factory.get_network_fn(
name=FLAGS.feature_extractor_name, num_classes=None)
default_image_size = getattr(net_fn, 'default_image_size', 224)
# Build tensorflow graph.
g = tf.Graph()
with g.as_default():
input_node = tf.placeholder(tf.uint8, shape=(None, None, None, 3))
images = tf.image.convert_image_dtype(input_node, dtype=tf.float32)
images = tf.image.resize_bilinear(
images, size=(default_image_size, default_image_size))
images = tf.subtract(images, 0.5)
images = tf.multiply(images, 2.0)
prediction, end_points = net_fn(images)
prediction = tf.squeeze(prediction, [1, 2])
# The init_fn function.
init_fn = slim.assign_from_checkpoint_fn(
FLAGS.feature_extractor_checkpoint,
slim.get_model_variables(FLAGS.feature_extractor_name))
uninitialized_variable_names = tf.report_uninitialized_variables()
# Start session to extract video features.
with tf.Session(graph=g) as sess:
init_fn(sess)
warn_names = sess.run(uninitialized_variable_names)
assert len(warn_names) == 0
def _extract_feature(video_id, images_data):
values = sess.run(prediction, feed_dict={ input_node: images_data })
return values
# Iterate through video ids.
for video_id in _video_id_iterator(FLAGS.video_id_path):
_extract_video_feature(video_id, FLAGS.video_dir,
FLAGS.output_dir, _extract_feature)
logging.info('Done')
def main(_):
if not FLAGS.output_file:
raise ValueError(
'You must supply the path to save to with --output_file')
tf.logging.set_verbosity(tf.logging.INFO)
with tf.Graph().as_default() as graph:
dataset = dataset_factory.get_dataset(FLAGS.dataset_name, 'train',
FLAGS.dataset_dir)
network_fn = nets_factory.get_network_fn(
FLAGS.model_name,
num_classes=(dataset.num_classes - FLAGS.labels_offset),
is_training=FLAGS.is_training)
image_size = FLAGS.image_size or network_fn.default_image_size
placeholder = tf.placeholder(
name='input',
dtype=tf.float32,
shape=[FLAGS.batch_size, image_size, image_size, 3])
network_fn(placeholder)
graph_def = graph.as_graph_def()
with gfile.GFile(FLAGS.output_file, 'wb') as f:
f.write(graph_def.SerializeToString())
def main(_):
if not FLAGS.output_file:
raise ValueError(
'You must supply the path to save to with --output_file')
if FLAGS.is_video_model and not FLAGS.num_frames:
raise ValueError(
'Number of frames must be specified for video models with --num_frames'
)
tf.logging.set_verbosity(tf.logging.INFO)
with tf.Graph().as_default() as graph:
dataset = dataset_factory.get_dataset(FLAGS.dataset_name, 'train',
FLAGS.dataset_dir)
network_fn = nets_factory.get_network_fn(
FLAGS.model_name,
num_classes=(dataset.num_classes - FLAGS.labels_offset),
is_training=FLAGS.is_training)
image_size = FLAGS.image_size or network_fn.default_image_size
if FLAGS.is_video_model:
input_shape = [
FLAGS.batch_size, FLAGS.num_frames, image_size, image_size, 3
]
else:
input_shape = [FLAGS.batch_size, image_size, image_size, 3]
placeholder = tf.placeholder(name='input',
dtype=tf.float32,
shape=input_shape)
network_fn(placeholder)
if FLAGS.quantize:
tf.contrib.quantize.create_eval_graph()
graph_def = graph.as_graph_def()
if FLAGS.write_text_graphdef:
tf.io.write_graph(graph_def,
os.path.dirname(FLAGS.output_file),
os.path.basename(FLAGS.output_file),
as_text=True)
else:
with gfile.GFile(FLAGS.output_file, 'wb') as f:
f.write(graph_def.SerializeToString())
def build_imagenet_graph(path):
tf.reset_default_graph()
print(path)
filename_queue = tf.train.string_input_producer(
tf.train.match_filenames_once(path + "/*.jpg"),
num_epochs=1,
shuffle=False,
capacity=100)
image_reader = tf.WholeFileReader()
image_file_name, image_file = image_reader.read(filename_queue)
image = tf.image.decode_jpeg(image_file, channels=3, fancy_upscaling=True)
model_name = 'inception_resnet_v2'
network_fn = nets_factory.get_network_fn(model_name,
is_training=False,
num_classes=1001)
preprocessing_name = model_name
image_preprocessing_fn = preprocessing_factory.get_preprocessing(
preprocessing_name, is_training=False)
eval_image_size = network_fn.default_image_size
image = image_preprocessing_fn(image, eval_image_size, eval_image_size)
filenames, images = tf.train.batch([image_file_name, image],
batch_size=100,
num_threads=2,
capacity=500)
logits, _ = network_fn(images)
variables_to_restore = slim.get_variables_to_restore()
predictions = tf.argmax(logits, 1)
return filenames, logits, predictions, variables_to_restore
def main(_):
if not FLAGS.dataset_dir:
raise ValueError(
'You must supply the dataset directory with --dataset_dir')
tf.logging.set_verbosity(tf.logging.INFO)
with tf.Graph().as_default():
tf_global_step = slim.get_or_create_global_step()
######################
# Select the dataset #
######################
dataset = dataset_factory.get_dataset(FLAGS.dataset_name,
FLAGS.dataset_split_name,
FLAGS.dataset_dir)
####################
# Select the model #
####################
network_fn = nets_factory.get_network_fn(
FLAGS.model_name,
num_classes=(dataset.num_classes - FLAGS.labels_offset),
is_training=False)
##############################################################
# Create a dataset provider that loads data from the dataset #
##############################################################
provider = slim.dataset_data_provider.DatasetDataProvider(
dataset,
shuffle=False,
common_queue_capacity=2 * FLAGS.batch_size,
common_queue_min=FLAGS.batch_size)
[image, label] = provider.get(['image', 'label'])
label -= FLAGS.labels_offset
#####################################
# Select the preprocessing function #
#####################################
preprocessing_name = FLAGS.preprocessing_name or FLAGS.model_name
image_preprocessing_fn = preprocessing_factory.get_preprocessing(
preprocessing_name, is_training=False)
eval_image_size = FLAGS.eval_image_size or network_fn.default_image_size
image = image_preprocessing_fn(image, eval_image_size, eval_image_size)
images, labels = tf.train.batch(
[image, label],
batch_size=FLAGS.batch_size,
num_threads=FLAGS.num_preprocessing_threads,
capacity=5 * FLAGS.batch_size)
####################
# Define the model #
####################
logits, _ = network_fn(images)
if FLAGS.moving_average_decay:
variable_averages = tf.train.ExponentialMovingAverage(
FLAGS.moving_average_decay, tf_global_step)
variables_to_restore = variable_averages.variables_to_restore(
slim.get_model_variables())
variables_to_restore[tf_global_step.op.name] = tf_global_step
else:
variables_to_restore = slim.get_variables_to_restore()
predictions = tf.argmax(logits, 1)
labels = tf.squeeze(labels)
# Define the metrics:
names_to_values, names_to_updates = slim.metrics.aggregate_metric_map({
'Accuracy':
slim.metrics.streaming_accuracy(predictions, labels),
'Recall_5':
slim.metrics.streaming_recall_at_k(logits, labels, 5),
})
# Print the summaries to screen.
for name, value in names_to_values.items():
summary_name = 'eval/%s' % name
op = tf.summary.scalar(summary_name, value, collections=[])
op = tf.Print(op, [value], summary_name)
tf.add_to_collection(tf.GraphKeys.SUMMARIES, op)
# TODO(sguada) use num_epochs=1
if FLAGS.max_num_batches:
num_batches = FLAGS.max_num_batches
else:
# This ensures that we make a single pass over all of the data.
num_batches = math.ceil(dataset.num_samples /
float(FLAGS.batch_size))
if tf.gfile.IsDirectory(FLAGS.checkpoint_path):
checkpoint_path = tf.train.latest_checkpoint(FLAGS.checkpoint_path)
else:
checkpoint_path = FLAGS.checkpoint_path
tf.logging.info('Evaluating %s' % checkpoint_path)
slim.evaluation.evaluate_once(
master=FLAGS.master,
checkpoint_path=checkpoint_path,
logdir=FLAGS.eval_dir,
num_evals=num_batches,
eval_op=list(names_to_updates.values()),
variables_to_restore=variables_to_restore)
def main(_):
if not FLAGS.dataset_dir:
raise ValueError(
'You must supply the dataset directory with --dataset_dir')
tf.logging.set_verbosity(tf.logging.INFO)
with tf.Graph().as_default():
#######################
# Config model_deploy #
#######################
deploy_config = model_deploy.DeploymentConfig(
num_clones=FLAGS.num_clones,
clone_on_cpu=FLAGS.clone_on_cpu,
replica_id=FLAGS.task,
num_replicas=FLAGS.worker_replicas,
num_ps_tasks=FLAGS.num_ps_tasks)
# Create global_step
with tf.device(deploy_config.variables_device()):
global_step = slim.create_global_step()
######################
# Select the dataset #
######################
dataset = dataset_factory.get_dataset(FLAGS.dataset_name,
FLAGS.dataset_split_name,
FLAGS.dataset_dir)
######################
# Select the network #
######################
network_fn = nets_factory.get_network_fn(
FLAGS.model_name,
num_classes=(dataset.num_classes - FLAGS.labels_offset),
weight_decay=FLAGS.weight_decay,
is_training=True)
#####################################
# Select the preprocessing function #
#####################################
preprocessing_name = FLAGS.preprocessing_name or FLAGS.model_name
image_preprocessing_fn = preprocessing_factory.get_preprocessing(
preprocessing_name, is_training=True)
##############################################################
# Create a dataset provider that loads data from the dataset #
##############################################################
with tf.device(deploy_config.inputs_device()):
provider = slim.dataset_data_provider.DatasetDataProvider(
dataset,
num_readers=FLAGS.num_readers,
common_queue_capacity=20 * FLAGS.batch_size,
common_queue_min=10 * FLAGS.batch_size)
[image, label] = provider.get(['image', 'label'])
label -= FLAGS.labels_offset
train_image_size = FLAGS.train_image_size or network_fn.default_image_size
image = image_preprocessing_fn(image, train_image_size,
train_image_size)
images, labels = tf.train.batch(
[image, label],
batch_size=FLAGS.batch_size,
num_threads=FLAGS.num_preprocessing_threads,
capacity=5 * FLAGS.batch_size)
labels = slim.one_hot_encoding(
labels, dataset.num_classes - FLAGS.labels_offset)
batch_queue = slim.prefetch_queue.prefetch_queue(
[images, labels], capacity=2 * deploy_config.num_clones)
####################
# Define the model #
####################
def clone_fn(batch_queue):
"""Allows data parallelism by creating multiple clones of network_fn."""
images, labels = batch_queue.dequeue()
logits, end_points = network_fn(images)
#############################
# Specify the loss function #
#############################
if 'AuxLogits' in end_points:
slim.losses.softmax_cross_entropy(
end_points['AuxLogits'],
labels,
label_smoothing=FLAGS.label_smoothing,
weights=0.4,
scope='aux_loss')
slim.losses.softmax_cross_entropy(
logits,
labels,
label_smoothing=FLAGS.label_smoothing,
weights=1.0)
return end_points
# Gather initial summaries.
summaries = set(tf.get_collection(tf.GraphKeys.SUMMARIES))
clones = model_deploy.create_clones(deploy_config, clone_fn,
[batch_queue])
first_clone_scope = deploy_config.clone_scope(0)
# Gather update_ops from the first clone. These contain, for example,
# the updates for the batch_norm variables created by network_fn.
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS,
first_clone_scope)
# Add summaries for end_points.
end_points = clones[0].outputs
for end_point in end_points:
x = end_points[end_point]
summaries.add(tf.summary.histogram('activations/' + end_point, x))
summaries.add(
tf.summary.scalar('sparsity/' + end_point,
tf.nn.zero_fraction(x)))
# Add summaries for losses.
for loss in tf.get_collection(tf.GraphKeys.LOSSES, first_clone_scope):
summaries.add(tf.summary.scalar('losses/%s' % loss.op.name, loss))
# Add summaries for variables.
for variable in slim.get_model_variables():
summaries.add(tf.summary.histogram(variable.op.name, variable))
#################################
# Configure the moving averages #
#################################
if FLAGS.moving_average_decay:
moving_average_variables = slim.get_model_variables()
variable_averages = tf.train.ExponentialMovingAverage(
FLAGS.moving_average_decay, global_step)
else:
moving_average_variables, variable_averages = None, None
#########################################
# Configure the optimization procedure. #
#########################################
with tf.device(deploy_config.optimizer_device()):
learning_rate = _configure_learning_rate(dataset.num_samples,
global_step)
optimizer = _configure_optimizer(learning_rate)
summaries.add(tf.summary.scalar('learning_rate', learning_rate))
if FLAGS.sync_replicas:
# If sync_replicas is enabled, the averaging will be done in the chief
# queue runner.
optimizer = tf.train.SyncReplicasOptimizer(
opt=optimizer,
replicas_to_aggregate=FLAGS.replicas_to_aggregate,
total_num_replicas=FLAGS.worker_replicas,
variable_averages=variable_averages,
variables_to_average=moving_average_variables)
elif FLAGS.moving_average_decay:
# Update ops executed locally by trainer.
update_ops.append(
variable_averages.apply(moving_average_variables))
# Variables to train.
variables_to_train = _get_variables_to_train()
# and returns a train_tensor and summary_op
total_loss, clones_gradients = model_deploy.optimize_clones(
clones, optimizer, var_list=variables_to_train)
# Add total_loss to summary.
summaries.add(tf.summary.scalar('total_loss', total_loss))
# Create gradient updates.
grad_updates = optimizer.apply_gradients(clones_gradients,
global_step=global_step)
update_ops.append(grad_updates)
update_op = tf.group(*update_ops)
with tf.control_dependencies([update_op]):
train_tensor = tf.identity(total_loss, name='train_op')
# Add the summaries from the first clone. These contain the summaries
# created by model_fn and either optimize_clones() or _gather_clone_loss().
summaries |= set(
tf.get_collection(tf.GraphKeys.SUMMARIES, first_clone_scope))
# Merge all summaries together.
summary_op = tf.summary.merge(list(summaries), name='summary_op')
###########################
# Kicks off the training. #
###########################
slim.learning.train(
train_tensor,
logdir=FLAGS.train_dir,
master=FLAGS.master,
is_chief=(FLAGS.task == 0),
init_fn=_get_init_fn(),
summary_op=summary_op,
number_of_steps=FLAGS.max_number_of_steps,
log_every_n_steps=FLAGS.log_every_n_steps,
save_summaries_secs=FLAGS.save_summaries_secs,
save_interval_secs=FLAGS.save_interval_secs,
sync_optimizer=optimizer if FLAGS.sync_replicas else None)
def run_transfer_learning(root_model_dir, bot_model_dir, protobuf_dir, model_name='inception_v4',
dataset_split_name='train',
dataset_name='bot',
checkpoint_exclude_scopes=None,
trainable_scopes=None,
max_train_time_sec=None,
max_number_of_steps=None,
log_every_n_steps=None,
save_summaries_secs=None,
optimization_params=None):
"""
Starts the transfer learning of a model in a tensorflow session
:param root_model_dir: Directory containing the root models pretrained checkpoint files
:param bot_model_dir: Directory where the transfer learned model's checkpoint files are written to
:param protobuf_dir: Directory for the dataset factory to load the bot's training data from
:param model_name: name of the network model for the net factory to provide the correct network and preprocesing fn
:param dataset_split_name: 'train' or 'validation'
:param dataset_name: triggers the dataset factory to load a bot dataset
:param checkpoint_exclude_scopes: Layers to exclude when restoring the models variables
:param trainable_scopes: Layers to train from the restored model
:param max_train_time_sec: time boundary to stop training after in seconds
:param max_number_of_steps: maximum number of steps to run
:param log_every_n_steps: write a log after every nth optimization step
:param save_summaries_secs: save summaries to disc every n seconds
:param optimization_params: parameters for the optimization
:return:
"""
if not optimization_params:
optimization_params = OPTIMIZATION_PARAMS
if not max_number_of_steps:
max_number_of_steps = _MAX_NUMBER_OF_STEPS
if not checkpoint_exclude_scopes:
checkpoint_exclude_scopes = _CHECKPOINT_EXCLUDE_SCOPES
if not trainable_scopes:
trainable_scopes = _TRAINABLE_SCOPES
if not max_train_time_sec:
max_train_time_sec = _MAX_TRAIN_TIME_SECONDS
if not log_every_n_steps:
log_every_n_steps = _LOG_EVERY_N_STEPS
if not save_summaries_secs:
save_summaries_secs = _SAVE_SUMMARRIES_SECS
tf.logging.set_verbosity(tf.logging.INFO)
with tf.Graph().as_default():
#######################
# Config model_deploy #
#######################
deploy_config = model_deploy.DeploymentConfig(
num_clones=_NUM_CLONES,
clone_on_cpu=_CLONE_ON_CPU,
replica_id=_TASK,
num_replicas=_WORKER_REPLICAS,
num_ps_tasks=_NUM_PS_TASKS)
# Create global_step
with tf.device(deploy_config.variables_device()):
global_step = slim.create_global_step()
######################
# Select the dataset #
######################
dataset = dataset_factory.get_dataset(
dataset_name, dataset_split_name, protobuf_dir)
######################
# Select the network #
######################
network_fn = nets_factory.get_network_fn(
model_name,
num_classes=(dataset.num_classes - _LABELS_OFFSET),
weight_decay=OPTIMIZATION_PARAMS['weight_decay'],
is_training=True,
dropout_keep_prob=OPTIMIZATION_PARAMS['dropout_keep_prob'])
#####################################
# Select the preprocessing function #
#####################################
image_preprocessing_fn = preprocessing_factory.get_preprocessing(
model_name,
is_training=True)
##############################################################
# Create a dataset provider that loads data from the dataset #
##############################################################
with tf.device(deploy_config.inputs_device()):
provider = slim.dataset_data_provider.DatasetDataProvider(
dataset,
num_readers=_NUM_READERS,
common_queue_capacity=20 * _BATCH_SIZE,
common_queue_min=10 * _BATCH_SIZE)
[image, label] = provider.get(['image', 'label'])
label -= _LABELS_OFFSET
train_image_size = network_fn.default_image_size
image = image_preprocessing_fn(image, train_image_size, train_image_size)
images, labels = tf.train.batch(
[image, label],
batch_size=_BATCH_SIZE,
num_threads=_NUM_PREPROCESSING_THREADS,
capacity=5 * _BATCH_SIZE)
labels = slim.one_hot_encoding(
labels, dataset.num_classes - _LABELS_OFFSET)
batch_queue = slim.prefetch_queue.prefetch_queue(
[images, labels], capacity=2 * deploy_config.num_clones)
####################
# Define the model #
####################
def clone_fn(batch_queue):
"""Allows data parallelism by creating multiple clones of network_fn."""
images, labels = batch_queue.dequeue()
logits, end_points = network_fn(images)
#############################
# Specify the loss function #
#############################
if 'AuxLogits' in end_points:
tf.losses.softmax_cross_entropy(
logits=end_points['AuxLogits'], onehot_labels=labels,
label_smoothing=_LABEL_SMOOTHING, weights=0.4, scope='aux_loss')
tf.losses.softmax_cross_entropy(
logits=logits, onehot_labels=labels,
label_smoothing=_LABEL_SMOOTHING, weights=1.0)
return end_points
# Gather initial summaries.
summaries = set(tf.get_collection(tf.GraphKeys.SUMMARIES))
clones = model_deploy.create_clones(deploy_config, clone_fn, [batch_queue])
first_clone_scope = deploy_config.clone_scope(0)
# Gather update_ops from the first clone. These contain, for example,
# the updates for the batch_norm variables created by network_fn.
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS, first_clone_scope)
# Add summaries for end_points.
end_points = clones[0].outputs
for end_point in end_points:
x = end_points[end_point]
summaries.add(tf.summary.histogram('activations/' + end_point, x))
summaries.add(tf.summary.scalar('sparsity/' + end_point,
tf.nn.zero_fraction(x)))
# Add summaries for losses.
for loss in tf.get_collection(tf.GraphKeys.LOSSES, first_clone_scope):
summaries.add(tf.summary.scalar('losses/%s' % loss.op.name, loss))
# Add summaries for variables.
for variable in slim.get_model_variables():
summaries.add(tf.summary.histogram(variable.op.name, variable))
#################################
# Configure the moving averages #
#################################
if OPTIMIZATION_PARAMS['moving_average_decay']:
moving_average_variables = slim.get_model_variables()
variable_averages = tf.train.ExponentialMovingAverage(
OPTIMIZATION_PARAMS['moving_average_decay'], global_step)
else:
moving_average_variables, variable_averages = None, None
#########################################
# Configure the optimization procedure. #
#########################################
with tf.device(deploy_config.optimizer_device()):
learning_rate = _configure_learning_rate(dataset.num_samples, global_step)
optimizer = _configure_optimizer(learning_rate)
summaries.add(tf.summary.scalar('learning_rate', learning_rate))
if _SYNC_REPLICAS:
# If sync_replicas is enabled, the averaging will be done in the chief
# queue runner.
optimizer = tf.train.SyncReplicasOptimizer(
opt=optimizer,
replicas_to_aggregate=_REPLICAS_TO_AGGREGATE,
variable_averages=variable_averages,
variables_to_average=moving_average_variables,
replica_id=tf.constant(_TASK, tf.int32, shape=()),
total_num_replicas=_WORKER_REPLICAS)
elif OPTIMIZATION_PARAMS['moving_average_decay']:
# Update ops executed locally by trainer.
update_ops.append(variable_averages.apply(moving_average_variables))
# Variables to train.
variables_to_train = _get_variables_to_train(trainable_scopes)
# and returns a train_tensor and summary_op
total_loss, clones_gradients = model_deploy.optimize_clones(
clones,
optimizer,
var_list=variables_to_train)
# Add total_loss to summary.
summaries.add(tf.summary.scalar('total_loss', total_loss))
# Create gradient updates.
grad_updates = optimizer.apply_gradients(clones_gradients,
global_step=global_step)
update_ops.append(grad_updates)
update_op = tf.group(*update_ops)
train_tensor = control_flow_ops.with_dependencies([update_op], total_loss,
name='train_op')
# Add the summaries from the first clone. These contain the summaries
# created by model_fn and either optimize_clones() or _gather_clone_loss().
summaries |= set(tf.get_collection(tf.GraphKeys.SUMMARIES,
first_clone_scope))
# Merge all summaries together.
summary_op = tf.summary.merge(list(summaries), name='summary_op')
###########################
# Kicks off the training. #
###########################
slim.learning.train(
train_tensor,
logdir=bot_model_dir,
train_step_fn=train_step, # Manually added a custom train step to stop after max_time
train_step_kwargs=_train_step_kwargs(logdir=bot_model_dir, max_train_time_seconds=max_train_time_sec),
master=_MASTER,
is_chief=(_TASK == 0),
init_fn=_get_init_fn(root_model_dir, bot_model_dir, checkpoint_exclude_scopes),
summary_op=summary_op,
# number_of_steps=max_number_of_steps,
log_every_n_steps=log_every_n_steps,
save_summaries_secs=save_summaries_secs,
save_interval_secs=_SAVE_INTERNAL_SECS,
sync_optimizer=optimizer if _SYNC_REPLICAS else None)
def RCNN(inputs, proposals, options, is_training=True):
"""Runs RCNN model on the `inputs`.
Args:
inputs: Input image, a [batch, height, width, 3] uint8 tensor. The pixel
values are in the range of [0, 255].
proposals: Boxes used to crop the image features, using normalized
coordinates. It should be a [batch, max_num_proposals, 4] float tensor
denoting [y1, x1, y2, x2].
options: A fast_rcnn_pb2.FastRCNN proto.
is_training: If true, the model shall be executed in training mode.
Returns:
A [batch, max_num_proposals, feature_dims] tensor.
Raises:
ValueError if options is invalid.
"""
if not isinstance(options, rcnn_pb2.RCNN):
raise ValueError('The options has to be a rcnn_pb2.RCNN proto!')
if inputs.dtype != tf.uint8:
raise ValueError('The inputs has to be a tf.uint8 tensor.')
net_fn = nets_factory.get_network_fn(name=options.feature_extractor_name,
num_classes=1001)
default_image_size = getattr(net_fn, 'default_image_size', 224)
# Preprocess image.
preprocess_fn = preprocessing_factory.get_preprocessing(
options.feature_extractor_name, is_training=False)
inputs = preprocess_fn(inputs,
output_height=None,
output_width=None,
crop_image=False)
# Crop and resize images.
batch = proposals.shape[0]
max_num_proposals = tf.shape(proposals)[1]
box_ind = tf.expand_dims(tf.range(batch), axis=-1)
box_ind = tf.tile(box_ind, [1, max_num_proposals])
cropped_inputs = tf.image.crop_and_resize(
inputs,
boxes=tf.reshape(proposals, [-1, 4]),
box_ind=tf.reshape(box_ind, [-1]),
crop_size=[default_image_size, default_image_size])
# Run CNN.
_, end_points = net_fn(cropped_inputs)
outputs = end_points[options.feature_extractor_endpoint]
outputs = tf.reshape(outputs, [batch, max_num_proposals, -1])
init_fn = slim.assign_from_checkpoint_fn(
options.feature_extractor_checkpoint,
slim.get_model_variables(options.feature_extractor_scope))
def _init_from_ckpt_fn(_, sess):
return init_fn(sess)
return outputs, _init_from_ckpt_fn
def eval(bot_id,
bot_suffix,
setting_id=None,
dataset_split='train',
dataset_name='bot',
model_name='inception_v4',
preprocessing=None,
moving_average_decay=None,
tf_master=''):
full_id = bot_id + bot_suffix
if setting_id:
protobuf_dir = dirs.get_transfer_proto_dir(bot_id, setting_id)
else:
protobuf_dir = dirs.get_protobuf_dir(bot_id)
_check_dir(protobuf_dir)
print("READIND FROM %s" % (protobuf_dir))
performance_data_dir = dirs.get_performance_data_dir(bot_id)
# if os.listdir(performance_data_dir):
# raise ValueError('%s is not empty' % performance_data_dir)
tf.logging.set_verbosity(tf.logging.INFO)
with tf.Graph().as_default():
tf_global_step = slim.get_or_create_global_step()
######################
# Select the dataset #
######################
dataset = dataset_factory.get_dataset(dataset_name, dataset_split,
protobuf_dir)
####################
# Select the model #
####################
network_fn = nets_factory.get_network_fn(
model_name,
num_classes=(dataset.num_classes - LABELS_OFFSET),
is_training=False)
##############################################################
# Create a dataset provider that loads data from the dataset #
##############################################################
provider = slim.dataset_data_provider.DatasetDataProvider(
dataset,
shuffle=False,
common_queue_capacity=2 * BATCH_SIZE,
common_queue_min=BATCH_SIZE)
[image, label] = provider.get(['image', 'label'])
label -= LABELS_OFFSET
#####################################
# Select the preprocessing function #
#####################################
preprocessing_name = preprocessing or model_name
image_preprocessing_fn = preprocessing_factory.get_preprocessing(
preprocessing_name, is_training=False)
eval_image_size = EVAL_IMAGE_SIZE or network_fn.default_image_size
image = image_preprocessing_fn(image, eval_image_size, eval_image_size)
images, labels = tf.train.batch([image, label],
batch_size=BATCH_SIZE,
num_threads=NUM_THREADS,
capacity=5 * BATCH_SIZE)
####################
# Define the model #
####################
logits, _ = network_fn(images)
if moving_average_decay:
variable_averages = tf.train.ExponentialMovingAverage(
moving_average_decay, tf_global_step)
variables_to_restore = variable_averages.variables_to_restore(
slim.get_model_variables())
variables_to_restore[tf_global_step.op.name] = tf_global_step
else:
variables_to_restore = slim.get_variables_to_restore()
predictions = tf.argmax(logits, 1)
labels = tf.squeeze(labels)
# Define the metrics:
names_to_values, names_to_updates = slim.metrics.aggregate_metric_map({
'Accuracy':
slim.metrics.streaming_accuracy(predictions, labels),
'Recall_5':
slim.metrics.streaming_recall_at_k(logits, labels, 5),
})
# Print the summaries to screen.
for name, value in names_to_values.items():
summary_name = 'eval/%s' % name
op = tf.summary.scalar(summary_name, value, collections=[])
op = tf.Print(op, [value], summary_name)
tf.add_to_collection(tf.GraphKeys.SUMMARIES, op)
# TODO(sguada) use num_epochs=1
if MAX_NUM_BATCHES:
num_batches = MAX_NUM_BATCHES
else:
# This ensures that we make a single pass over all of the data.
num_batches = math.ceil(dataset.num_samples / float(BATCH_SIZE))
print(dataset.num_samples)
print(dataset.num_classes)
def main(_):
logging.set_verbosity(logging.INFO)
examples = _load_annots(FLAGS.bounding_box_json)
logging.info('Loaded %s examples.', len(examples))
# Create computational graph.
g = tf.Graph()
with g.as_default():
# Create model.
net_fn = nets_factory.get_network_fn(name=FLAGS.feature_extractor_name,
num_classes=1001)
default_image_size = getattr(net_fn, 'default_image_size', 224)
images = tf.placeholder(shape=(None, default_image_size,
default_image_size, 3),
dtype=tf.float32)
_, end_points = net_fn(images)
output_tensor = end_points[FLAGS.feature_extractor_endpoint]
init_fn = slim.assign_from_checkpoint_fn(
FLAGS.feature_extractor_checkpoint,
slim.get_model_variables(FLAGS.feature_extractor_scope))
uninitialized_variable_names = tf.report_uninitialized_variables()
# Start session.
results = {}
with tf.Session(graph=g, config=default_session_config()) as sess:
init_fn(sess)
assert len(sess.run(uninitialized_variable_names)) == 0
for index, (image_id, example) in enumerate(examples.iteritems()):
if index % 50 == 0:
logging.info('On image %i/%i', index, len(examples))
# Load image, preprocess. TODO: now only works for inception family.
filename = "{}/{}".format(FLAGS.image_dir, image_id)
bgr = cv2.imread(filename, cv2.IMREAD_COLOR)
rgb = bgr[:, :, ::-1]
rgb = rgb.astype(np.float32) * 2.0 / 255.0 - 1.0
# Batch operation.
regions_list = []
for region in example['regions'][:FLAGS.max_number_of_regions]:
roi = image_crop_and_resize(
rgb,
bbox=(region['bbox']['xmin'], region['bbox']['ymin'],
region['bbox']['xmax'], region['bbox']['ymax']),
crop_size=(default_image_size, default_image_size))
regions_list.append(roi)
batch = np.stack(regions_list, axis=0)
# Save results.
features = sess.run(output_tensor, feed_dict={images: batch})
results[image_id] = features
# Write results.
assert len(results) == len(examples)
with open(FLAGS.output_feature_path, 'wb') as fp:
np.save(fp, results)
logging.info('Exported features for %i images.', len(results))
logging.info('Done')
def main(_):
logging.set_verbosity(logging.INFO)
examples = _load_image_path_list(FLAGS.image_data_path)
logging.info('Load %s examples.', len(examples))
# Create computational graph.
g = tf.Graph()
with g.as_default():
net_fn = nets_factory.get_network_fn(name=FLAGS.feature_extractor_name,
num_classes=1001)
default_image_size = getattr(net_fn, 'default_image_size', 224)
images = tf.placeholder(shape=(None, default_image_size,
default_image_size, 3),
dtype=tf.float32)
_, end_points = net_fn(images)
output_tensor = end_points[FLAGS.feature_extractor_endpoint]
init_fn = slim.assign_from_checkpoint_fn(
FLAGS.feature_extractor_checkpoint,
slim.get_model_variables(FLAGS.feature_extractor_scope))
uninitialized_variable_names = tf.report_uninitialized_variables()
# Start session.
with tf.Session(graph=g) as sess:
init_fn(sess)
assert len(sess.run(uninitialized_variable_names)) == 0
for index, (image_id, filename) in enumerate(examples):
if index % 10 == 0:
logging.info('On image %i/%i', index, len(examples))
# Load bounding boxes.
data = _load_bounding_box_annotations(image_id,
FLAGS.bounding_box_json_path)
# Load image, preprocess.
bgr = cv2.imread(filename, cv2.IMREAD_COLOR)
rgb = bgr[:, :, ::-1].astype(np.float32) * 2.0 / 255.0 - 1.0
# Batch all the ROIs within an image.
rois = []
for paragraph in data['paragraphs'][:FLAGS.max_number_of_regions]:
roi = _crop_and_resize(
rgb,
bounding_box=(paragraph['bounding_box']['ymin'],
paragraph['bounding_box']['xmin'],
paragraph['bounding_box']['ymax'],
paragraph['bounding_box']['xmax']),
crop_size=(default_image_size, default_image_size))
rois.append(roi)
batch = np.stack(rois, axis=0)
# Save to file.
with open(
os.path.join(FLAGS.nmsed_bounding_box_json_path,
'{}.json'.format(image_id)), 'w') as fid:
fid.write(json.dumps(data, indent=2))
if FLAGS.nmsed_roi_image_path:
for i, roi in enumerate(rois):
dirname = os.path.join(FLAGS.nmsed_roi_image_path,
'{}'.format(image_id % 10))
os.makedirs(dirname, exist_ok=True)
filename = os.path.join(dirname,
'{}_{}.jpg'.format(image_id, i))
roi_bgr = ((roi[:, :, ::-1] + 1) / 2 * 255).astype(
np.uint8)
cv2.imwrite(filename, roi_bgr)
features = sess.run(output_tensor, feed_dict={images: batch})
_save(image_id, features, FLAGS.feature_output_path)
logging.info('Done')
def main(_):
logging.set_verbosity(logging.INFO)
examples = _load_annots(FLAGS.action_reason_annot_path)
logging.info('Loaded %s examples.', len(examples))
# Create computational graph.
g = tf.Graph()
with g.as_default():
# Create model.
net_fn = nets_factory.get_network_fn(name=FLAGS.feature_extractor_name,
num_classes=1001)
default_image_size = getattr(net_fn, 'default_image_size', 224)
images = tf.placeholder(shape=(None, default_image_size,
default_image_size, 3),
dtype=tf.float32)
_, end_points = net_fn(images)
output_tensor = end_points[FLAGS.feature_extractor_endpoint]
init_fn = slim.assign_from_checkpoint_fn(
FLAGS.feature_extractor_checkpoint,
slim.get_model_variables(FLAGS.feature_extractor_scope))
uninitialized_variable_names = tf.report_uninitialized_variables()
# Start session.
results = {}
with tf.Session(graph=g, config=default_session_config()) as sess:
init_fn(sess)
assert len(sess.run(uninitialized_variable_names)) == 0
image_ids, batch = [], []
for index, (image_id, example) in enumerate(examples.iteritems()):
# Process the current batch.
if index % FLAGS.batch_size == 0:
if len(batch) > 0:
features = sess.run(
output_tensor,
feed_dict={images: np.stack(batch, axis=0)})
for img_id, feature in zip(image_ids, features):
results[img_id] = feature
image_ids, batch = [], []
logging.info('On image %i/%i', index, len(examples))
# Load image, preprocess. TODO: now only works for inception family.
filename = "{}/{}".format(FLAGS.image_dir, image_id)
bgr = cv2.imread(filename, cv2.IMREAD_COLOR)
rgb = bgr[:, :, ::-1]
rgb = rgb.astype(np.float32) * 2.0 / 255.0 - 1.0
rgb = cv2.resize(rgb, (default_image_size, default_image_size))
batch.append(rgb)
image_ids.append(image_id)
# For the final batch.
if len(batch) > 0:
features = sess.run(output_tensor,
feed_dict={images: np.stack(batch, axis=0)})
for image_id, feature in zip(image_ids, features):
results[image_id] = feature
# Write results.
assert len(results) == len(examples)
with open(FLAGS.output_feature_path, 'wb') as fp:
np.save(fp, results)
logging.info('Exported features for %i images.', len(results))
logging.info('Done')
