def main(unused):
g = tf.Graph()
with g.as_default(), tf.device('/cpu:0'):
# dataset = EsosData(subset='train',
# datadir='./data/bbox/ESOS',
# num_train_imgs=10966,
# num_test_imgs=2741)
dataset = Pascal(subset='trainval',
datadir='./data/bbox/VOC2007selected0410_0000',
num_train_imgs=3328,
num_test_imgs=5011)
images, filenames, sizes, labels, bbox, roi_labels, gt_labels, gt = \
image_processing.batch_inputs(dataset,
train=FLAGS.is_training,
flip_image=FLAGS.flip_images,
crop_image=FLAGS.crop_images,
image_size=400,
batch_size=1)
valid_rois = tf.reduce_sum(
tf.to_float(tf.greater(bbox[:, :, 2] - bbox[:, :, 0], 1e-6)))
with tf.Session() as sess:
tf.train.start_queue_runners(sess=sess)
for step in range(10):
print(step)
images_out, filenames_out, sizes_out, roi_labels_out, gt_labels_out, gts_out, bbox_out, labels_out, valid_rois_out = \
sess.run([images, filenames, sizes, roi_labels, gt_labels, gt, bbox, labels, valid_rois])
print(filenames_out)
print(labels_out)
print(sizes_out)
for i in range(len(dataset.category_list())):
if labels_out[0, i] > 0.5:
print(dataset.category_list()[i])
for i in range(len(dataset.category_list())):
if roi_labels_out[0, i] > 0.5:
print(dataset.category_list()[i])
print(bbox_out[0, 100])
print(gt_labels_out)
print(bbox_out[:, 0:5, :])
print('num rois: %d' % valid_rois_out)
gt_out = (400 * gts_out[0]).astype(np.int32)
bbox_out = (400 * bbox_out[0]).astype(np.int32)
gt_vis = visualize_boxes(images_out[0], gt_out)
roi_vis = visualize_boxes(images_out[0], bbox_out[:10, :])
cv2.imshow('img', gt_vis)
cv2.imshow('rois', roi_vis)
cv2.waitKey()
def main(unused_args):
dataset = CarsData(subset=FLAGS.subset)
assert dataset.data_files()
with tf.Graph().as_default():
# Get images and labels from the dataset.
images, labels = image_processing.batch_inputs(
dataset=dataset,
batch_size=FLAGS.batch_size,
num_preprocess_threads=FLAGS.num_preprocess_threads,
train=False,
regular=True)
images_NCHW = tf.transpose(images, (0, 3, 1, 2))
# Build a Graph that computes the logits predictions from the
# inference model.
print "Setting up model"
logits, predictions = model.inference(images_NCHW)
# Calculate predictions.
top_1_op = tf.nn.in_top_k(logits, (labels - 1), 1)
# Build the summary operation based on the TF collection of Summaries.
summary_op = tf.summary.merge_all()
graph_def = tf.get_default_graph().as_graph_def()
summary_writer = tf.summary.FileWriter(FLAGS.eval_dir,
graph_def=graph_def)
saver = tf.train.Saver()
with tf.Session() as sess:
ckpt = tf.train.get_checkpoint_state(FLAGS.checkpoint_dir)
if ckpt and ckpt.model_checkpoint_path:
if os.path.isabs(ckpt.model_checkpoint_path):
# Restores from checkpoint with absolute path.
saver.restore(sess, ckpt.model_checkpoint_path)
else:
# Restores from checkpoint with relative path.
saver.restore(
sess,
os.path.join(FLAGS.checkpoint_dir,
ckpt.model_checkpoint_path))
# Assuming model_checkpoint_path looks something like:
# /my-favorite-path/imagenet_train/model.ckpt-0,
# extract global_step from it.
global_step = ckpt.model_checkpoint_path.split('/')[-1].split(
'-')[-1]
print('Successfully loaded model from %s at step=%s.' %
(ckpt.model_checkpoint_path, global_step))
else:
print('No checkpoint file found')
return
# Start the queue runners.
coord = tf.train.Coordinator()
try:
threads = []
for qr in tf.get_collection(tf.GraphKeys.QUEUE_RUNNERS):
threads.extend(
qr.create_threads(sess,
coord=coord,
daemon=True,
start=True))
num_iter = int(math.ceil(FLAGS.num_examples /
FLAGS.batch_size))
# Counts the number of correct predictions.
count_top_1 = 0.0
total_sample_count = num_iter * FLAGS.batch_size
step = 0
print('%s: starting evaluation on (%s).' %
(datetime.now(), FLAGS.subset))
start_time = time.time()
while step < num_iter and not coord.should_stop():
top_1 = sess.run([top_1_op])
count_top_1 += np.sum(top_1)
step += 1
if step % 20 == 0:
duration = time.time() - start_time
sec_per_batch = duration / 20.0
examples_per_sec = FLAGS.batch_size / sec_per_batch
print(
'%s: [%d batches out of %d] (%.1f examples/sec; %.3f'
'sec/batch)' % (datetime.now(), step, num_iter,
examples_per_sec, sec_per_batch))
start_time = time.time()
# Compute precision @ 1
precision_at_1 = count_top_1 / total_sample_count
print('%s: precision @ 1 = %.4f [%d examples]' %
(datetime.now(), precision_at_1, total_sample_count))
summary = tf.Summary()
summary.ParseFromString(sess.run(summary_op))
summary.value.add(tag='Precision @ 1',
simple_value=precision_at_1)
summary_writer.add_summary(summary, global_step)
except Exception as e: # pylint: disable=broad-except
coord.request_stop(e)
coord.request_stop()
coord.join(threads, stop_grace_period_secs=10)
def worker_train(model_path,
data_dir,
batch_size,
preprocess_operation,
image_size,
num_classes,
train_steps,
val_stpes,
val_interval,
objectives,
do_evaluation=False,
labels_offset=0,
num_preprocess_threads=None,
num_readers=1,
examples_per_shard=1024,
input_queue_memory_factor=16):
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
K.set_session(sess)
model = load_model(model_path)
print("loading tensorflow data reading queues...")
train_image_tensor, train_labels_tensor, train_labels_text_tensor = image_processing.batch_inputs(
data_dir, batch_size, image_size, True, preprocess_operation,
num_preprocess_threads, num_readers, examples_per_shard,
input_queue_memory_factor)
val_image_tensor, val_labels_tensor, test_labels_text_tensor = image_processing.batch_inputs(
data_dir, batch_size, image_size, False, preprocess_operation,
num_preprocess_threads, num_readers, examples_per_shard,
input_queue_memory_factor)
tf.train.start_queue_runners(sess=sess)
train_labels = train_labels_tensor
val_labels = val_labels_tensor
if labels_offset != 0:
train_labels = tf.sub(train_labels, labels_offset)
val_labels = tf.sub(val_labels, labels_offset)
# check objectives
if objectives == 'categorical_crossentropy':
train_labels = tf.one_hot(train_labels, num_classes, axis=-1)
train_labels = tf.cast(train_labels, tf.float32)
val_labels = tf.one_hot(val_labels, num_classes, axis=-1)
val_labels = tf.cast(val_labels, tf.float32)
elif objectives == 'binary_crossentropy':
train_labels = tf.cast(train_labels, tf.float32)
train_labels = tf.reshape(train_labels, [-1, 1])
val_labels = tf.cast(val_labels, tf.float32)
val_labels = tf.reshape(val_labels, [-1, 1])
else:
print('No corresponding objectives %s' % objectives)
exit(-1)
print("training...")
with sess.as_default():
for i in range(1, train_steps + 1):
train_x, train_y = sess.run([train_image_tensor, train_labels])
print get_time(), model.train_on_batch(train_x, train_y)
if i % val_interval == 0 and do_evaluation:
val_res = []
for j in range(val_stpes):
val_x, val_y = sess.run([val_image_tensor, val_labels])
val_res.append(model.test_on_batch(val_x, val_y))
print get_time(), "val: ", np.mean(val_res, 0).tolist()
cur_dir_path = os.path.dirname(os.path.realpath(__file__))
save_path = os.path.join(cur_dir_path, 'trained_model.h5')
model.save(save_path)
def worker_train(model_path,
data_dir,
batch_size,
preprocess_operation,
image_size,
num_classes,
max_steps,
objectives,
optimizers,
learning_rate=0.01,
labels_offset=0,
num_preprocess_threads=None,
num_readers=1,
examples_per_shard=1024,
input_queue_memory_factor=16):
sess = tf.Session()
K.set_session(sess)
model = load_model(model_path)
train_image_tensor, train_labels_tensor = image_processing.batch_inputs(
data_dir, batch_size, image_size, True, preprocess_operation,
num_preprocess_threads, num_readers, examples_per_shard,
input_queue_memory_factor)
#x = tf.placeholder(tf.float32, shape=(None, image_size[0], image_size[1], image_size[2]))
y = tf.placeholder(tf.int32, shape=(batch_size, ))
#preds = model(x)
x = model.input
preds = model.output
train_labels = y
if labels_offset != 0:
train_labels = tf.sub(y, labels_offset)
# check objectives
if objectives == 'categorical_crossentropy':
train_labels = tf.one_hot(train_labels, num_classes, axis=-1)
train_labels = tf.cast(train_labels, tf.float32)
loss = tf.reduce_mean(categorical_crossentropy(train_labels, preds))
# this loss is the same as:
# loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(preds, train_labels))
acc_value = categorical_accuracy(train_labels, preds)
elif objectives == 'binary_crossentropy':
# train_labels: 1-d tensor (batch_size,)
# preds: 2-d tensor (batch_size, 1)
train_labels = tf.cast(train_labels, tf.float32)
preds = tf.reshape(preds, train_labels.get_shape().as_list())
# or reshape train_labels as:
# train_labels = tf.reshape(train_labels, [-1,1])
loss = tf.reduce_mean(binary_crossentropy(train_labels, preds))
# this loss is the same as:
# loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(preds, train_labels))
acc_value = binary_accuracy(train_labels, preds)
else:
print('No corresponding objectives %s' % objectives)
exit(-1)
print(preds.get_shape().as_list())
print(train_labels.get_shape().as_list())
# check optimizers
if optimizers == 'RMSProp':
# recommend learning rate: around 0.001
train_step = tf.train.RMSPropOptimizer(learning_rate).minimize(loss)
elif optimizers == 'GradientDescent':
# recommend learning rate: around 0.01
train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(
loss)
else:
print('No corresponding optimizers %s' % optimizers)
exit(-1)
sess.run(tf.initialize_all_variables())
tf.train.start_queue_runners(sess=sess)
with sess.as_default():
for i in range(max_steps):
tx, ty = sess.run([train_image_tensor, train_labels_tensor])
_, cur_loss, cur_accu = sess.run([train_step, loss, acc_value],
feed_dict={
K.learning_phase(): 1,
x: tx,
y: ty
})
print cur_loss, cur_accu
with open('json_model.json', 'w') as f:
f.write(model.to_json())
model.save_weights('json_model_weights.h5')
def worker_test(data_dir,
batch_size,
preprocess_operation,
image_size,
num_classes,
max_steps,
objectives,
optimizers,
learning_rate=0.01,
labels_offset=0,
num_preprocess_threads=None,
num_readers=1,
examples_per_shard=1024,
input_queue_memory_factor=16):
sess = tf.Session()
K.set_session(sess)
with open('json_model.json') as f:
json_str = f.readline().strip()
model = model_from_json(json_str)
train_image_tensor, train_labels_tensor = image_processing.batch_inputs(
data_dir, batch_size, image_size, True, preprocess_operation,
num_preprocess_threads, num_readers, examples_per_shard,
input_queue_memory_factor)
y = tf.placeholder(tf.int32, shape=(batch_size, ))
x = model.input
preds = model.output
train_labels = y
if labels_offset != 0:
train_labels = tf.sub(y, labels_offset)
# check objectives
if objectives == 'categorical_crossentropy':
train_labels = tf.one_hot(train_labels, num_classes, axis=-1)
train_labels = tf.cast(train_labels, tf.float32)
loss = tf.reduce_mean(categorical_crossentropy(train_labels, preds))
# this loss is the same as:
# loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(preds, train_labels))
acc_value = categorical_accuracy(train_labels, preds)
elif objectives == 'binary_crossentropy':
# train_labels: 1-d tensor (batch_size,)
# preds: 2-d tensor (batch_size, 1)
train_labels = tf.cast(train_labels, tf.float32)
preds = tf.reshape(preds, train_labels.get_shape().as_list())
# or reshape train_labels as:
# train_labels = tf.reshape(train_labels, [-1,1])
loss = tf.reduce_mean(binary_crossentropy(train_labels, preds))
# this loss is the same as:
# loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(preds, train_labels))
acc_value = binary_accuracy(train_labels, preds)
else:
print('No corresponding objectives %s' % objectives)
exit(-1)
print(preds.get_shape().as_list())
print(train_labels.get_shape().as_list())
sess.run(tf.initialize_all_variables())
model.load_weights('json_model_weights.h5')
tf.train.start_queue_runners(sess=sess)
with sess.as_default():
for i in range(max_steps):
tx, ty = sess.run([train_image_tensor, train_labels_tensor])
cur_loss, cur_accu = sess.run([loss, acc_value],
feed_dict={
K.learning_phase(): 0,
x: tx,
y: ty
})
print cur_loss, cur_accu
def main(unused_args):
if FLAGS.force_use_cpu:
os.environ['CUDA_VISIBLE_DEVICES'] = ''
ps_hosts = FLAGS.ps_hosts.split(',')
worker_hosts = FLAGS.worker_hosts.split(',')
tf.logging.info('Worker hosts are: %s' % worker_hosts)
cluster_spec = tf.train.ClusterSpec({
'ps': ps_hosts,
'worker': worker_hosts
})
server = tf.train.Server(cluster_spec.as_dict(),
job_name=FLAGS.job_name,
task_index=FLAGS.task_id,
protocol=FLAGS.protocol)
if FLAGS.job_name == 'ps':
print "I'm a parameter server."
server.join()
else:
dataset = CarsData(subset=FLAGS.subset)
assert dataset.data_files()
if FLAGS.task_id == 0:
if not tf.gfile.Exists(FLAGS.train_dir):
tf.gfile.MakeDirs(FLAGS.train_dir)
with tf.device(
tf.train.replica_device_setter(
worker_device="/job:worker/task:%d" % FLAGS.task_id,
cluster=cluster_spec)):
print "I'm a worker"
images_placeholder = tf.placeholder(dtype=tf.float32,
shape=(None, 3, 224, 224),
name="images_placeholder")
labels_placeholder = tf.placeholder(dtype=tf.int32,
shape=(None),
name="labels_placeholder")
images, labels = image_processing.batch_inputs(
dataset=dataset,
batch_size=FLAGS.batch_size,
num_preprocess_threads=FLAGS.num_preprocess_threads,
train=True,
regular=True)
logits, predictions = model.inference(images_placeholder)
loss = model.loss(logits, labels_placeholder)
accuracy = model.accuracy(logits, labels_placeholder)
global_step = tf.contrib.framework.get_or_create_global_step()
opt = tf.train.GradientDescentOptimizer(learning_rate=0.05)
num_workers = len(cluster_spec.as_dict()['worker'])
print "Number of workers: %d" % num_workers
opt = tf.train.SyncReplicasOptimizer(
opt,
replicas_to_aggregate=num_workers,
total_num_replicas=num_workers,
use_locking=True,
name='sync_replicas')
train_op = opt.minimize(loss=loss, global_step=global_step)
is_chief = (FLAGS.task_id == 0)
print "IsChief: %s" % is_chief
sync_replicas_hook = opt.make_session_run_hook(
is_chief=is_chief, num_tokens=num_workers)
last_step_hook = tf.train.StopAtStepHook(num_steps=FLAGS.max_steps)
hooks = [sync_replicas_hook, last_step_hook]
with tf.train.MonitoredTrainingSession(
config=tf.ConfigProto(
allow_soft_placement=True,
log_device_placement=FLAGS.log_device_placement),
master=server.target,
is_chief=is_chief,
checkpoint_dir=FLAGS.train_dir,
hooks=hooks,
stop_grace_period_secs=120) as mon_session:
while not mon_session.should_stop():
image_feed, label_feed = mon_session.run([images, labels])
# Need to check mon_session.should_stop after each session.run to avoid
# errors when calling run after stop
if (mon_session.should_stop()):
break
# Convert from NHWC to NCHW format
image_feed_NCHW = np.transpose(image_feed, (0, 3, 1, 2))
feed_dict = {
images_placeholder: image_feed_NCHW,
labels_placeholder: label_feed
}
print "==================================================="
print "Running train op"
_, current_loss, current_step, current_accuracy = \
mon_session.run([train_op, loss, global_step, accuracy],
feed_dict = feed_dict)
print "Current step: %s" % current_step
print "Current loss: %.2f" % current_loss
print "Current accuracy: %.4f" % current_accuracy
print "==================================================="