def main(_):
logging.set_verbosity(tf.logging.INFO)
assert os.path.isfile(FLAGS.pipeline_proto)
g = tf.Graph()
with g.as_default():
pipeline_proto = load_pipeline_proto(FLAGS.pipeline_proto)
logging.info("Pipeline configure: %s", '=' * 128)
logging.info(pipeline_proto)
# Get examples from reader.
examples = ads_examples.get_examples(pipeline_proto.example_reader)
# Build model for evaluation.
global_step = slim.get_or_create_global_step()
model = builder.build(pipeline_proto.model, is_training=False)
predictions = model.build_inference(examples)
loss_dict = model.build_loss(predictions)
uninitialized_variable_names = tf.report_uninitialized_variables()
saver = tf.train.Saver()
init_op = tf.group(tf.local_variables_initializer(),
tf.global_variables_initializer())
session_config = train_utils.default_session_config(
FLAGS.per_process_gpu_memory_fraction)
# Start session.
logging.info('=' * 128)
eval_config = pipeline_proto.eval_config
model_path = train_utils.get_latest_model(FLAGS.saved_ckpts_dir)
if model_path is not None:
with tf.Session(graph=g, config=session_config) as sess:
# Initialize model.
sess.run(init_op)
saver.restore(sess, model_path)
logging.info('Restore from %s.', model_path)
warn_names = sess.run(uninitialized_variable_names)
assert len(warn_names) == 0
# Evaluation loop.
step = sess.run(global_step)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
evaluate_once(sess,
None,
step,
examples['video_id'],
predictions=predictions)
coord.request_stop()
coord.join(threads, stop_grace_period_secs=10)
logging.info('Done')
def main(_):
logging.set_verbosity(logging.INFO)
examples = _load_annots(FLAGS.bounding_box_json)
logging.info('Loaded %s examples.', len(examples))
# Create computational graph.
g = tf.Graph()
with g.as_default():
# Create model.
net_fn = nets_factory.get_network_fn(name=FLAGS.feature_extractor_name,
num_classes=1001)
default_image_size = getattr(net_fn, 'default_image_size', 224)
images = tf.placeholder(shape=(None, default_image_size,
default_image_size, 3),
dtype=tf.float32)
_, end_points = net_fn(images)
output_tensor = end_points[FLAGS.feature_extractor_endpoint]
init_fn = slim.assign_from_checkpoint_fn(
FLAGS.feature_extractor_checkpoint,
slim.get_model_variables(FLAGS.feature_extractor_scope))
uninitialized_variable_names = tf.report_uninitialized_variables()
# Start session.
results = {}
with tf.Session(graph=g, config=default_session_config()) as sess:
init_fn(sess)
assert len(sess.run(uninitialized_variable_names)) == 0
for index, (image_id, example) in enumerate(examples.iteritems()):
if index % 50 == 0:
logging.info('On image %i/%i', index, len(examples))
# Load image, preprocess. TODO: now only works for inception family.
filename = "{}/{}".format(FLAGS.image_dir, image_id)
bgr = cv2.imread(filename, cv2.IMREAD_COLOR)
rgb = bgr[:, :, ::-1]
rgb = rgb.astype(np.float32) * 2.0 / 255.0 - 1.0
# Batch operation.
regions_list = []
for region in example['regions'][:FLAGS.max_number_of_regions]:
roi = image_crop_and_resize(
rgb,
bbox=(region['bbox']['xmin'], region['bbox']['ymin'],
region['bbox']['xmax'], region['bbox']['ymax']),
crop_size=(default_image_size, default_image_size))
regions_list.append(roi)
batch = np.stack(regions_list, axis=0)
# Save results.
features = sess.run(output_tensor, feed_dict={images: batch})
results[image_id] = features
# Write results.
assert len(results) == len(examples)
with open(FLAGS.output_feature_path, 'wb') as fp:
np.save(fp, results)
logging.info('Exported features for %i images.', len(results))
logging.info('Done')
def main(_):
logging.set_verbosity(tf.logging.INFO)
assert os.path.isfile(FLAGS.pipeline_proto)
assert os.path.isfile(FLAGS.action_reason_annot_path)
pipeline_proto = load_pipeline_proto(FLAGS.pipeline_proto)
logging.info("Pipeline configure: %s", '=' * 128)
logging.info(pipeline_proto)
groundtruths = load_action_reason_annots(FLAGS.action_reason_annot_path)
g = tf.Graph()
with g.as_default():
# Get examples from reader.
split = 'valid'
if not FLAGS.continuous_evaluation:
split = 'test'
examples, feed_init_fn = ads_mem_examples.get_examples(
pipeline_proto.example_reader, split)
# Build model for training.
global_step = slim.get_or_create_global_step()
model = builder.build(pipeline_proto.model, is_training=False)
predictions = model.build_evaluation_graph(examples)
init_fn = model.get_init_fn()
uninitialized_variable_names = tf.report_uninitialized_variables()
saver = tf.train.Saver()
init_op = tf.group(tf.local_variables_initializer(),
tf.global_variables_initializer())
session_config = train_utils.default_session_config(
FLAGS.per_process_gpu_memory_fraction)
# evaluation on test set.
logging.info('Start evaluating.')
eval_config = pipeline_proto.eval_config
# One time evaluation and inference.
if not FLAGS.continuous_evaluation:
model_path = train_utils.get_latest_model(FLAGS.saved_ckpt_dir)
with tf.Session(graph=g, config=session_config) as sess:
feed_init_fn(sess)
sess.run(init_op)
saver.restore(sess, model_path)
logging.info('Restore model from %s.', model_path)
warn_names = sess.run(uninitialized_variable_names)
assert len(warn_names) == 0
# Evaluate the best model in terms of recall@3.
step = sess.run(global_step)
evaluate_once(sess, None, step, predictions, groundtruths)
logging.info('Done')
exit(0)
# Continuous evaluation on valid set.
writer = tf.summary.FileWriter(FLAGS.eval_log_dir, g)
step = prev_step = -1
while True:
start = time.time()
try:
model_path = tf.train.latest_checkpoint(FLAGS.train_log_dir)
if model_path is not None:
with tf.Session(graph=g, config=session_config) as sess:
# Restore model.
feed_init_fn(sess)
sess.run(init_op)
saver.restore(sess, model_path)
logging.info('Restore model from %s.', model_path)
warn_names = sess.run(uninitialized_variable_names)
assert len(warn_names) == 0
step = sess.run(global_step)
if step != prev_step and step > eval_config.eval_min_global_steps:
# Evaluate the latest model.
prev_step = step
metric = evaluate_once(sess, writer, step, predictions,
groundtruths)
step_best, metric_best = train_utils.save_model_if_it_is_better(
step,
metric,
model_path,
FLAGS.saved_ckpt_dir,
reverse=False)
if step_best == step:
summary = tf.Summary()
summary.value.add(tag='metrics/model_metric',
simple_value=metric_best)
writer.add_summary(summary, global_step=step)
writer.flush()
# with tf.Session
# if model_path is not None
except Exception as ex:
pass
if step >= eval_config.number_of_steps:
break
sleep_secs = eval_config.eval_interval_secs - (time.time() - start)
if sleep_secs > 0:
logging.info('Now sleep for %.2lf secs.', sleep_secs)
time.sleep(sleep_secs)
writer.close()
logging.info('Done')
def main(_):
logging.set_verbosity(tf.logging.INFO)
model_proto = _load_model_proto(FLAGS.model_proto)
logging.info('Model proto: \n%s', model_proto)
# Load vocab.
word_to_id, id_to_symbol = load_symbol_cluster(FLAGS.symbol_cluster_path)
logging.info('Number of classes: %i.', len(id_to_symbol))
# Load image features.
img_features = np.load(FLAGS.feature_path).item()
logging.info('Load %i features.', len(img_features))
# Note that ZERO is reserved for 'unclear'.
annots = load_raw_annots(FLAGS.symbol_annot_path)
x, y = _get_data(annots, img_features, len(id_to_symbol))
number_of_val_examples = FLAGS.number_of_val_examples
x_train, y_train = x[number_of_val_examples:], y[number_of_val_examples:]
x_valid, y_valid = x[:number_of_val_examples], y[:number_of_val_examples]
logging.info('Load %d train examples.', len(x_train))
logging.info('Load %d valid examples.', len(x_valid))
# Build graph to train symbol classifier.
g = tf.Graph()
with g.as_default():
# For training
logits, init_fn = mlp.model(model_proto, x_train, is_training=True)
loss_op = tf.nn.sigmoid_cross_entropy_with_logits(labels=y_train,
logits=logits)
loss_op = tf.reduce_mean(loss_op)
optimizer = tf.train.AdamOptimizer(FLAGS.learning_rate)
train_op = slim.learning.create_train_op(loss_op, optimizer=optimizer)
init_op = tf.global_variables_initializer()
# For evaluation.
tf.get_variable_scope().reuse_variables()
logits_val, init_fn_val = mlp.model(model_proto,
x_valid,
is_training=False)
saver = tf.train.Saver()
with tf.Session(graph=g,
config=train_utils.default_session_config()) as sess:
sess.run(init_op)
max_v, metrics = 0, []
for i in xrange(FLAGS.max_iters):
(_, loss, pred_train,
pred_valid) = sess.run([train_op, loss_op, logits, logits_val])
mAP_micro = average_precision_score(y_valid[:, 1:],
pred_valid[:, 1:],
average='micro')
mAP_macro = average_precision_score(y_valid[:, 1:],
pred_valid[:, 1:],
average='macro')
metric = mAP_macro
if i % 100 == 0:
logging.info(
'step=%d, loss=%.4lf, mAP_micro=%.4lf, mAP_macro=%.4lf.',
i + 1, loss, mAP_micro, mAP_macro)
if metric >= max_v:
saver.save(sess, FLAGS.output_model_path)
max_v = metric
if len(metrics) >= 3:
if metric < metrics[-1] and metrics[-1] < metrics[
-2] and metrics[-2] < metrics[-3]:
logging.info('Process early stopping.')
break
metrics.append(metric)
logging.info('Done')
def main(_):
logging.set_verbosity(tf.logging.INFO)
assert os.path.isfile(FLAGS.pipeline_proto)
assert os.path.isfile(FLAGS.sentiment_vocab_path)
assert os.path.isfile(FLAGS.sentiment_anno_vocab)
assert os.path.isfile(FLAGS.sentiment_raw_annot_path)
assert os.path.isfile(FLAGS.sentiment_clean_annot_path)
sentiment_raw_annot = load_raw_annot(FLAGS.sentiment_raw_annot_path)
sentiment_clean_annot = eval_utils.load_clean_annot(FLAGS.sentiment_clean_annot_path)
topic_clean_annot = load_clean_annot(FLAGS.topic_clean_annot_path)
g = tf.Graph()
with g.as_default():
pipeline_proto = load_pipeline_proto(FLAGS.pipeline_proto)
logging.info("Pipeline configure: %s", '=' * 128)
logging.info(pipeline_proto)
# Get examples from reader.
examples = ads_examples.get_examples(pipeline_proto.example_reader)
# Build model for evaluation.
global_step = slim.get_or_create_global_step()
model = builder.build(pipeline_proto.model, is_training=False)
predictions = model.build_inference(examples)
loss_dict = model.build_loss(predictions)
uninitialized_variable_names = tf.report_uninitialized_variables()
saver = tf.train.Saver()
init_op = tf.group(tf.local_variables_initializer(),
tf.global_variables_initializer())
session_config = train_utils.default_session_config(
FLAGS.per_process_gpu_memory_fraction)
# Start session.
logging.info('=' * 128)
eval_config = pipeline_proto.eval_config
writer = tf.summary.FileWriter(FLAGS.eval_log_dir, g)
prev_step = -1
while True:
start = time.time()
try:
model_path = tf.train.latest_checkpoint(FLAGS.train_log_dir)
if model_path is not None:
with tf.Session(graph=g, config=session_config) as sess:
# Initialize model.
sess.run(init_op)
saver.restore(sess, model_path)
logging.info('Restore from %s.', model_path)
warn_names = sess.run(uninitialized_variable_names)
assert len(warn_names) == 0
step = sess.run(global_step)
if step != prev_step and step > eval_config.eval_min_global_steps:
prev_step = step
# Evaluation loop.
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
model_metric = evaluate_sentiment_once(sess,
writer,
step,
examples['video_id'],
predictions=predictions,
clean_annot=sentiment_clean_annot,
raw_annot=sentiment_raw_annot,
use_sigmoid=True)
step_best, metric_best = train_utils.save_model_if_it_is_better(
step, model_metric, model_path, FLAGS.saved_ckpts_dir,
reverse=False)
# We improved the model, record it.
if step_best == step:
summary = tf.Summary()
summary.value.add(
tag='metrics/metric_best', simple_value=metric_best)
writer.add_summary(summary, global_step=step)
writer.flush()
coord.request_stop()
coord.join(threads, stop_grace_period_secs=10)
if step >= eval_config.number_of_steps:
break
except Exception as ex:
pass
# Sleep a while.
sleep_secs = eval_config.eval_interval_secs - (time.time() - start)
if sleep_secs > 0:
logging.info('Now sleep for %.2lf secs.', sleep_secs)
time.sleep(sleep_secs)
writer.close()
logging.info('Done')
def main(_):
logging.set_verbosity(logging.INFO)
examples = _load_annots(FLAGS.action_reason_annot_path)
logging.info('Loaded %s examples.', len(examples))
# Create computational graph.
g = tf.Graph()
with g.as_default():
# Create model.
net_fn = nets_factory.get_network_fn(name=FLAGS.feature_extractor_name,
num_classes=1001)
default_image_size = getattr(net_fn, 'default_image_size', 224)
images = tf.placeholder(shape=(None, default_image_size,
default_image_size, 3),
dtype=tf.float32)
_, end_points = net_fn(images)
output_tensor = end_points[FLAGS.feature_extractor_endpoint]
init_fn = slim.assign_from_checkpoint_fn(
FLAGS.feature_extractor_checkpoint,
slim.get_model_variables(FLAGS.feature_extractor_scope))
uninitialized_variable_names = tf.report_uninitialized_variables()
# Start session.
results = {}
with tf.Session(graph=g, config=default_session_config()) as sess:
init_fn(sess)
assert len(sess.run(uninitialized_variable_names)) == 0
image_ids, batch = [], []
for index, (image_id, example) in enumerate(examples.iteritems()):
# Process the current batch.
if index % FLAGS.batch_size == 0:
if len(batch) > 0:
features = sess.run(
output_tensor,
feed_dict={images: np.stack(batch, axis=0)})
for img_id, feature in zip(image_ids, features):
results[img_id] = feature
image_ids, batch = [], []
logging.info('On image %i/%i', index, len(examples))
# Load image, preprocess. TODO: now only works for inception family.
filename = "{}/{}".format(FLAGS.image_dir, image_id)
bgr = cv2.imread(filename, cv2.IMREAD_COLOR)
rgb = bgr[:, :, ::-1]
rgb = rgb.astype(np.float32) * 2.0 / 255.0 - 1.0
rgb = cv2.resize(rgb, (default_image_size, default_image_size))
batch.append(rgb)
image_ids.append(image_id)
# For the final batch.
if len(batch) > 0:
features = sess.run(output_tensor,
feed_dict={images: np.stack(batch, axis=0)})
for image_id, feature in zip(image_ids, features):
results[image_id] = feature
# Write results.
assert len(results) == len(examples)
with open(FLAGS.output_feature_path, 'wb') as fp:
np.save(fp, results)
logging.info('Exported features for %i images.', len(results))
logging.info('Done')
def main(_):
logging.set_verbosity(tf.logging.INFO)
assert os.path.isfile(FLAGS.pipeline_proto)
g = tf.Graph()
with g.as_default():
pipeline_proto = load_pipeline_proto(FLAGS.pipeline_proto)
logging.info("Pipeline configure: %s", '=' * 128)
logging.info(pipeline_proto)
train_config = pipeline_proto.train_config
# Get examples from reader.
examples, feed_init_fn = ads_mem_examples.get_examples(
pipeline_proto.example_reader, split='train')
# Build model for training.
model = builder.build(pipeline_proto.model, is_training=True)
predictions = model.build_inference_graph(examples)
loss_dict = model.build_loss(predictions)
model_init_fn = model.get_init_fn()
uninitialized_variable_names = tf.report_uninitialized_variables()
if FLAGS.restore_from:
variables_to_restore = slim.get_variables_to_restore(
exclude=[name for name in train_config.exclude_variable])
restore_init_fn = slim.assign_from_checkpoint_fn(
FLAGS.restore_from, variables_to_restore)
def init_fn(sess):
model_init_fn(sess)
if FLAGS.restore_from:
restore_init_fn(sess)
# Loss and optimizer.
for loss_name, loss_tensor in loss_dict.iteritems():
tf.losses.add_loss(loss_tensor)
tf.summary.scalar('losses/{}'.format(loss_name), loss_tensor)
total_loss = tf.losses.get_total_loss()
tf.summary.scalar('losses/total_loss', total_loss)
for reg_loss in tf.losses.get_regularization_losses():
name = 'losses/reg_loss_{}'.format(reg_loss.op.name.split('/')[0])
tf.summary.scalar(name, reg_loss)
optimizer = train_utils.build_optimizer(train_config)
if train_config.moving_average:
optimizer = tf.contrib.opt.MovingAverageOptimizer(
optimizer, average_decay=0.99)
gradient_multipliers = train_utils.build_multipler(
train_config.gradient_multiplier)
variables_to_train = model.get_variables_to_train()
logging.info('=' * 128)
for var in variables_to_train:
logging.info(var)
train_op = slim.learning.create_train_op(
total_loss,
variables_to_train=variables_to_train,
clip_gradient_norm=0.0,
gradient_multipliers=gradient_multipliers,
summarize_gradients=True,
optimizer=optimizer)
saver = None
if train_config.moving_average:
saver = optimizer.swapping_saver()
# Start checking.
logging.info('Start checking...')
session_config = train_utils.default_session_config(
FLAGS.per_process_gpu_memory_fraction)
def _session_wrapper_fn(sess):
feed_init_fn(sess)
return sess
slim.learning.train(train_op,
logdir=FLAGS.train_log_dir,
graph=g,
master='',
is_chief=True,
number_of_steps=train_config.number_of_steps,
log_every_n_steps=train_config.log_every_n_steps,
save_interval_secs=train_config.save_interval_secs,
save_summaries_secs=train_config.save_summaries_secs,
session_config=session_config,
session_wrapper=_session_wrapper_fn,
init_fn=init_fn,
saver=saver)
logging.info('Done')
def main(_):
logging.set_verbosity(tf.logging.INFO)
assert os.path.isfile(FLAGS.pipeline_proto), FLAGS.pipeline_proto
g = tf.Graph()
with g.as_default():
pipeline_proto = load_pipeline_proto(FLAGS.pipeline_proto)
logging.info("Pipeline configure: %s", '=' * 128)
logging.info(pipeline_proto)
train_config = pipeline_proto.train_config
# Get examples from reader.
examples = ads_examples.get_examples(pipeline_proto.example_reader)
# Build model for training.
global_step = slim.get_or_create_global_step()
model = builder.build(pipeline_proto.model, is_training=True)
predictions = model.build_inference(examples)
loss_dict = model.build_loss(predictions)
init_fn = model.get_init_fn()
uninitialized_variable_names = tf.report_uninitialized_variables()
# Loss and optimizer.
for loss_name, loss_tensor in loss_dict.iteritems():
tf.losses.add_loss(loss_tensor)
tf.summary.scalar('losses/{}'.format(loss_name), loss_tensor)
total_loss = tf.losses.get_total_loss()
tf.summary.scalar('losses/total_loss', total_loss)
optimizer = train_utils.build_optimizer(train_config)
if train_config.moving_average:
optimizer = tf.contrib.opt.MovingAverageOptimizer(optimizer,
average_decay=0.99)
gradient_multipliers = train_utils.build_multipler(
train_config.gradient_multiplier)
variables_to_train = model.get_variables_to_train()
for var in variables_to_train:
logging.info(var)
train_op = slim.learning.create_train_op(total_loss,
variables_to_train=variables_to_train,
clip_gradient_norm=5.0,
gradient_multipliers=gradient_multipliers,
summarize_gradients=True,
optimizer=optimizer)
saver = None
if train_config.moving_average:
saver = optimizer.swapping_saver()
# Starts training.
logging.info('Start training.')
session_config = train_utils.default_session_config(
FLAGS.per_process_gpu_memory_fraction)
slim.learning.train(train_op,
logdir=FLAGS.train_log_dir,
graph=g,
master='',
is_chief=True,
number_of_steps=train_config.number_of_steps,
log_every_n_steps=train_config.log_every_n_steps,
save_interval_secs=train_config.save_interval_secs,
save_summaries_secs=train_config.save_summaries_secs,
session_config=session_config,
init_fn=init_fn,
saver=saver)
logging.info('Done')
