def freeze_graph(input_graph, input_saver, input_binary, input_checkpoint,
output_node_names, restore_op_name, filename_tensor_name,
output_graph, clear_devices, initializer_nodes):
"""Converts all variables in a graph and checkpoint into constants."""
if not tf.gfile.Exists(input_graph):
print("Input graph file '" + input_graph + "' does not exist!")
return -1
if input_saver and not tf.gfile.Exists(input_saver):
print("Input saver file '" + input_saver + "' does not exist!")
return -1
if not tf.gfile.Glob(input_checkpoint):
print("Input checkpoint '" + input_checkpoint + "' doesn't exist!")
return -1
if not output_node_names:
print("You need to supply the name of a node to --output_node_names.")
return -1
input_graph_def = tf.GraphDef()
mode = "rb" if input_binary else "r"
with tf.gfile.FastGFile(input_graph, mode) as f:
if input_binary:
input_graph_def.ParseFromString(f.read())
else:
text_format.Merge(f.read(), input_graph_def)
# Remove all the explicit device specifications for this node. This helps to
# make the graph more portable.
if clear_devices:
for node in input_graph_def.node:
node.device = ""
_ = tf.import_graph_def(input_graph_def, name="")
with tf.Session() as sess:
if input_saver:
with tf.gfile.FastGFile(input_saver, mode) as f:
saver_def = tf.train.SaverDef()
if input_binary:
saver_def.ParseFromString(f.read())
else:
text_format.Merge(f.read(), saver_def)
saver = tf.train.Saver(saver_def=saver_def)
saver.restore(sess, input_checkpoint)
else:
sess.run([restore_op_name],
{filename_tensor_name: input_checkpoint})
if initializer_nodes:
sess.run(initializer_nodes)
output_graph_def = graph_util.convert_variables_to_constants(
sess, input_graph_def, output_node_names.split(","))
with tf.gfile.GFile(output_graph, "wb") as f:
f.write(output_graph_def.SerializeToString())
print("%d ops in the final graph." % len(output_graph_def.node))
def testConvertVariablesToConsts(self):
with tf.Graph().as_default():
variable_node = tf.Variable(1.0, name="variable_node")
_ = tf.Variable(1.0, name="unused_variable_node")
output_node = tf.mul(variable_node, 2.0, name="output_node")
with tf.Session() as sess:
init = tf.initialize_variables([variable_node])
sess.run(init)
output = sess.run(output_node)
self.assertNear(2.0, output, 0.00001)
variable_graph_def = sess.graph.as_graph_def()
# First get the constant_graph_def when variable_names_whitelist is set,
# note that if variable_names_whitelist is not set an error will be
# thrown because unused_variable_node is not initialized.
constant_graph_def = graph_util.convert_variables_to_constants(
sess,
variable_graph_def, ["output_node"],
variable_names_whitelist=set(["variable_node"]))
# Then initialize the unused variable, and get another
# constant_graph_def when variable_names_whitelist is not set.
sess.run(tf.initialize_all_variables())
constant_graph_def_without_variable_whitelist = (
graph_util.convert_variables_to_constants(
sess, variable_graph_def, ["output_node"]))
# The unused variable should be cleared so the two graphs should be
# equivalent.
self.assertEqual(
str(constant_graph_def),
str(constant_graph_def_without_variable_whitelist))
# Now we make sure the variable is now a constant, and that the graph still
# produces the expected result.
with tf.Graph().as_default():
_ = tf.import_graph_def(constant_graph_def, name="")
self.assertEqual(4, len(constant_graph_def.node))
for node in constant_graph_def.node:
self.assertNotEqual("Variable", node.op)
with tf.Session() as sess:
output_node = sess.graph.get_tensor_by_name("output_node:0")
output = sess.run(output_node)
self.assertNear(2.0, output, 0.00001)
def freeze_graph(input_graph, input_saver, input_binary, input_checkpoint,
output_node_names, restore_op_name, filename_tensor_name,
output_graph, clear_devices, initializer_nodes):
"""Converts all variables in a graph and checkpoint into constants."""
if not tf.gfile.Exists(input_graph):
print("Input graph file '" + input_graph + "' does not exist!")
return -1
if input_saver and not tf.gfile.Exists(input_saver):
print("Input saver file '" + input_saver + "' does not exist!")
return -1
if not tf.gfile.Glob(input_checkpoint):
print("Input checkpoint '" + input_checkpoint + "' doesn't exist!")
return -1
if not output_node_names:
print("You need to supply the name of a node to --output_node_names.")
return -1
input_graph_def = tf.GraphDef()
mode = "rb" if input_binary else "r"
with tf.gfile.FastGFile(input_graph, mode) as f:
if input_binary:
input_graph_def.ParseFromString(f.read())
else:
text_format.Merge(f.read(), input_graph_def)
# Remove all the explicit device specifications for this node. This helps to
# make the graph more portable.
if clear_devices:
for node in input_graph_def.node:
node.device = ""
_ = tf.import_graph_def(input_graph_def, name="")
with tf.Session() as sess:
if input_saver:
with tf.gfile.FastGFile(input_saver, mode) as f:
saver_def = tf.train.SaverDef()
if input_binary:
saver_def.ParseFromString(f.read())
else:
text_format.Merge(f.read(), saver_def)
saver = tf.train.Saver(saver_def=saver_def)
saver.restore(sess, input_checkpoint)
else:
sess.run([restore_op_name], {filename_tensor_name: input_checkpoint})
if initializer_nodes:
sess.run(initializer_nodes)
output_graph_def = graph_util.convert_variables_to_constants(
sess, input_graph_def, output_node_names.split(","))
with tf.gfile.GFile(output_graph, "wb") as f:
f.write(output_graph_def.SerializeToString())
print("%d ops in the final graph." % len(output_graph_def.node))
def testConvertVariablesToConsts(self):
with tf.Graph().as_default():
variable_node = tf.Variable(1.0, name="variable_node")
_ = tf.Variable(1.0, name="unused_variable_node")
output_node = tf.mul(variable_node, 2.0, name="output_node")
with tf.Session() as sess:
init = tf.initialize_variables([variable_node])
sess.run(init)
output = sess.run(output_node)
self.assertNear(2.0, output, 0.00001)
variable_graph_def = sess.graph.as_graph_def()
# First get the constant_graph_def when variable_names_whitelist is set,
# note that if variable_names_whitelist is not set an error will be
# thrown because unused_variable_node is not initialized.
constant_graph_def = graph_util.convert_variables_to_constants(
sess, variable_graph_def, ["output_node"],
variable_names_whitelist=set(["variable_node"]))
# Then initialize the unused variable, and get another
# constant_graph_def when variable_names_whitelist is not set.
sess.run(tf.initialize_all_variables())
constant_graph_def_without_variable_whitelist = (
graph_util.convert_variables_to_constants(
sess, variable_graph_def, ["output_node"]))
# The unused variable should be cleared so the two graphs should be
# equivalent.
self.assertEqual(str(constant_graph_def),
str(constant_graph_def_without_variable_whitelist))
# Now we make sure the variable is now a constant, and that the graph still
# produces the expected result.
with tf.Graph().as_default():
_ = tf.import_graph_def(constant_graph_def, name="")
self.assertEqual(4, len(constant_graph_def.node))
for node in constant_graph_def.node:
self.assertNotEqual("Variable", node.op)
with tf.Session() as sess:
output_node = sess.graph.get_tensor_by_name("output_node:0")
output = sess.run(output_node)
self.assertNear(2.0, output, 0.00001)
def export():
with tf.Graph().as_default() as graph:
with tf.Session() as sess:
model = InceptionModel(ds)
model.build_graph(for_training=False)
model.load_last_checkpoint(sess)
output_file = os.path.join(MODEL_DIR, str(model) + ".pb")
graph_def = graph.as_graph_def()
graph_def = convert_variables_to_constants(sess, graph_def, ["results/predictions"])
with gfile.FastGFile(output_file, 'wb') as f:
f.write(graph_def.SerializeToString())
def freeze_graph(model_folder):
# We retrieve our checkpoint fullpath
checkpoint = tf.train.get_checkpoint_state(model_folder)
input_checkpoint = checkpoint.model_checkpoint_path
# We precise the file fullname of our freezed graph
absolute_model_folder = "/".join(input_checkpoint.split('/')[:-1])
output_graph = absolute_model_folder + "/frozen_model.pb"
# Before exporting our graph, we need to precise what is our output node
# This is how TF decides what part of the Graph he has to keep and what part it can dump
# NOTE: this variable is plural, because you can have multiple output nodes
output_node_names = "Accuracy/predictions"
# We clear devices to allow TensorFlow to control on which device it will load operations
clear_devices = True
# We import the meta graph and retrieve a Saver
saver = tf.train.import_meta_graph(input_checkpoint + '.meta',
clear_devices=clear_devices)
# We retrieve the protobuf graph definition
graph = tf.get_default_graph()
input_graph_def = graph.as_graph_def()
# We start a session and restore the graph weights
with tf.Session() as sess:
saver.restore(sess, input_checkpoint)
# We use a built-in TF helper to export variables to constants
output_graph_def = graph_util.convert_variables_to_constants(
sess, # The session is used to retrieve the weights
input_graph_def, # The graph_def is used to retrieve the nodes
output_node_names.split(
","
) # The output node names are used to select the usefull nodes
)
# Finally we serialize and dump the output graph to the filesystem
with tf.gfile.GFile(output_graph, "wb") as f:
f.write(output_graph_def.SerializeToString())
print("%d ops in the final graph." % len(output_graph_def.node))
def testConvertVariablesToConsts(self):
with tf.Graph().as_default():
variable_node = tf.Variable(1.0, name="variable_node")
output_node = tf.mul(variable_node, 2.0, name="output_node")
with tf.Session() as sess:
init = tf.initialize_all_variables()
sess.run(init)
output = sess.run(output_node)
self.assertNear(2.0, output, 0.00001)
variable_graph_def = sess.graph.as_graph_def()
constant_graph_def = graph_util.convert_variables_to_constants(
sess, variable_graph_def, ["output_node"])
# Now we make sure the variable is now a constant, and that the graph still
# produces the expected result.
with tf.Graph().as_default():
_ = tf.import_graph_def(constant_graph_def, name="")
self.assertEqual(4, len(constant_graph_def.node))
for node in constant_graph_def.node:
self.assertNotEqual("Variable", node.op)
with tf.Session() as sess:
output_node = sess.graph.get_tensor_by_name("output_node:0")
output = sess.run(output_node)
self.assertNear(2.0, output, 0.00001)
def testConvertVariablesToConsts(self):
with tf.Graph().as_default():
variable_node = tf.Variable(1.0, name="variable_node")
output_node = tf.mul(variable_node, 2.0, name="output_node")
with tf.Session() as sess:
init = tf.initialize_all_variables()
sess.run(init)
output = sess.run(output_node)
self.assertNear(2.0, output, 0.00001)
variable_graph_def = sess.graph.as_graph_def()
constant_graph_def = graph_util.convert_variables_to_constants(
sess, variable_graph_def, ["output_node"])
# Now we make sure the variable is now a constant, and that the graph still
# produces the expected result.
with tf.Graph().as_default():
_ = tf.import_graph_def(constant_graph_def, name="")
self.assertEqual(4, len(constant_graph_def.node))
for node in constant_graph_def.node:
self.assertNotEqual("Variable", node.op)
with tf.Session() as sess:
output_node = sess.graph.get_tensor_by_name("output_node:0")
output = sess.run(output_node)
self.assertNear(2.0, output, 0.00001)
def main(_):
# Set up the pre-trained graph.
maybe_download_and_extract()
graph, bottleneck_tensor, jpeg_data_tensor, resized_image_tensor = (
create_inception_graph())
# Look at the folder structure, and create lists of all the images.
image_lists = create_image_lists(FLAGS.image_dir, FLAGS.testing_percentage,
FLAGS.validation_percentage)
class_count = len(image_lists.keys())
if class_count == 0:
print('No valid folders of images found at ' + FLAGS.image_dir)
return -1
if class_count == 1:
print('Only one valid folder of images found at ' + FLAGS.image_dir +
' - multiple classes are needed for classification.')
return -1
# See if the command-line flags mean we're applying any distortions.
do_distort_images = should_distort_images(FLAGS.flip_left_right,
FLAGS.random_crop,
FLAGS.random_scale,
FLAGS.random_brightness)
sess = tf.Session()
if do_distort_images:
# We will be applying distortions, so setup the operations we'll need.
distorted_jpeg_data_tensor, distorted_image_tensor = add_input_distortions(
FLAGS.flip_left_right, FLAGS.random_crop, FLAGS.random_scale,
FLAGS.random_brightness)
else:
# We'll make sure we've calculated the 'bottleneck' image summaries and
# cached them on disk.
cache_bottlenecks(sess, image_lists, FLAGS.image_dir,
FLAGS.bottleneck_dir, jpeg_data_tensor,
bottleneck_tensor)
# Add the new layer that we'll be training.
(train_step, cross_entropy, bottleneck_input, ground_truth_input,
final_tensor) = add_final_training_ops(len(image_lists.keys()),
FLAGS.final_tensor_name,
bottleneck_tensor)
# Set up all our weights to their initial default values.
init = tf.initialize_all_variables()
sess.run(init)
# Create the operations we need to evaluate the accuracy of our new layer.
evaluation_step = add_evaluation_step(final_tensor, ground_truth_input)
# Run the training for as many cycles as requested on the command line.
for i in range(FLAGS.how_many_training_steps):
# Get a catch of input bottleneck values, either calculated fresh every time
# with distortions applied, or from the cache stored on disk.
if do_distort_images:
train_bottlenecks, train_ground_truth = get_random_distorted_bottlenecks(
sess, image_lists, FLAGS.train_batch_size, 'training',
FLAGS.image_dir, distorted_jpeg_data_tensor,
distorted_image_tensor, resized_image_tensor,
bottleneck_tensor)
else:
train_bottlenecks, train_ground_truth = get_random_cached_bottlenecks(
sess, image_lists, FLAGS.train_batch_size, 'training',
FLAGS.bottleneck_dir, FLAGS.image_dir, jpeg_data_tensor,
bottleneck_tensor)
# Feed the bottlenecks and ground truth into the graph, and run a training
# step.
sess.run(train_step,
feed_dict={
bottleneck_input: train_bottlenecks,
ground_truth_input: train_ground_truth
})
# Every so often, print out how well the graph is training.
is_last_step = (i + 1 == FLAGS.how_many_training_steps)
if (i % FLAGS.eval_step_interval) == 0 or is_last_step:
train_accuracy, cross_entropy_value = sess.run(
[evaluation_step, cross_entropy],
feed_dict={
bottleneck_input: train_bottlenecks,
ground_truth_input: train_ground_truth
})
print('%s: Step %d: Train accuracy = %.1f%%' %
(datetime.now(), i, train_accuracy * 100))
print('%s: Step %d: Cross entropy = %f' %
(datetime.now(), i, cross_entropy_value))
validation_bottlenecks, validation_ground_truth = (
get_random_cached_bottlenecks(
sess, image_lists, FLAGS.validation_batch_size,
'validation', FLAGS.bottleneck_dir, FLAGS.image_dir,
jpeg_data_tensor, bottleneck_tensor))
validation_accuracy = sess.run(evaluation_step,
feed_dict={
bottleneck_input:
validation_bottlenecks,
ground_truth_input:
validation_ground_truth
})
print('%s: Step %d: Validation accuracy = %.1f%%' %
(datetime.now(), i, validation_accuracy * 100))
# We've completed all our training, so run a final test evaluation on
# some new images we haven't used before.
test_bottlenecks, test_ground_truth = get_random_cached_bottlenecks(
sess, image_lists, FLAGS.test_batch_size, 'testing',
FLAGS.bottleneck_dir, FLAGS.image_dir, jpeg_data_tensor,
bottleneck_tensor)
test_accuracy = sess.run(evaluation_step,
feed_dict={
bottleneck_input: test_bottlenecks,
ground_truth_input: test_ground_truth
})
print('Final test accuracy = %.1f%%' % (test_accuracy * 100))
# Write out the trained graph and labels with the weights stored as constants.
output_graph_def = graph_util.convert_variables_to_constants(
sess, graph.as_graph_def(), [FLAGS.final_tensor_name])
with gfile.FastGFile(FLAGS.output_graph, 'wb') as f:
f.write(output_graph_def.SerializeToString())
with gfile.FastGFile(FLAGS.output_labels, 'w') as f:
f.write('\n'.join(image_lists.keys()) + '\n')
def main(_):
# Set up the pre-trained graph.
InputManager.maybe_download_and_extract(model_dir=FLAGS.model_dir,
data_url=DATA_URL)
(graph, bottleneck_tensor, jpeg_data_tensor,
resized_image_tensor) = GraphBuilder.create_inception_graph(
model_dir=FLAGS.model_dir,
bottleneck_tensor_name=BOTTLENECK_TENSOR_NAME,
jpeg_data_tensor_name=JPEG_DATA_TENSOR_NAME,
resized_input_tensor_name=RESIZED_INPUT_TENSOR_NAME)
# Look at the folder structure, and create lists of all the images.
image_lists = InputManager.create_image_lists(
image_dir=FLAGS.image_dir,
testing_percentage=FLAGS.testing_percentage,
validation_percentage=FLAGS.validation_percentage)
class_count = len(image_lists.keys())
if class_count == 0:
print('No valid folders of images found at ' + FLAGS.image_dir)
return -1
if class_count == 1:
print('Only one valid folder of images found at ' + FLAGS.image_dir +
' - multiple classes are needed for classification.')
return -1
# See if the command-line flags mean we're applying any distortions.
do_distort_images = utils.should_distort_images(
flip_left_right=FLAGS.flip_left_right,
random_crop=FLAGS.random_crop,
random_scale=FLAGS.random_scale,
random_brightness=FLAGS.random_brightness)
sess = tf.Session()
if do_distort_images:
# We will be applying distortions, so setup the operations we'll need.
(distorted_jpeg_data_tensor,
distorted_image_tensor) = GraphBuilder.add_input_distortions(
model_input_depth=MODEL_INPUT_DEPTH,
model_input_width=MODEL_INPUT_WIDTH,
model_input_height=MODEL_INPUT_HEIGHT,
flip_left_right=FLAGS.flip_left_right,
random_crop=FLAGS.random_crop,
random_scale=FLAGS.random_scale,
random_brightness=FLAGS.random_brightness)
else:
# We'll make sure we've calculated the 'bottleneck' image summaries and cached them on disk.
Bottleneck.cache_bottlenecks(sess=sess,
image_lists=image_lists,
image_dir=FLAGS.image_dir,
bottleneck_dir=FLAGS.bottleneck_dir,
jpeg_data_tensor=jpeg_data_tensor,
bottleneck_tensor=bottleneck_tensor)
# Add the new layer that we'll be training.
(train_step, cross_entropy, bottleneck_input, ground_truth_input,
final_tensor) = GraphBuilder.add_final_training_ops(
bottleneck_tensor_size=BOTTLENECK_TENSOR_SIZE,
learning_rate=FLAGS.learning_rate,
class_count=len(image_lists.keys()),
final_tensor_name=FLAGS.final_tensor_name,
bottleneck_tensor=bottleneck_tensor)
# Merge all the summaries and write them out to /tmp/transfer_learning_inception
merged = tf.merge_all_summaries()
train_writer = tf.train.SummaryWriter(FLAGS.summaries_dir + '/train',
sess.graph)
validation_writer = tf.train.SummaryWriter(FLAGS.summaries_dir +
'/validation')
# Set up all our weights to their initial default values.
init = tf.initialize_all_variables()
sess.run(init)
# Create the operations we need to evaluate the accuracy of our new layer.
evaluation_step = GraphBuilder.add_evaluation_step(final_tensor,
ground_truth_input)
# Run the training for as many cycles as requested on the command line.
for i in range(FLAGS.how_many_training_steps):
# Get a catch of input bottleneck values, either calculated fresh every time with distortions applied,
# or from the cache stored on disk.
if do_distort_images:
train_bottlenecks, train_ground_truth = Bottleneck.get_random_distorted_bottlenecks(
sess, image_lists, FLAGS.train_batch_size, 'training',
FLAGS.image_dir, distorted_jpeg_data_tensor,
distorted_image_tensor, resized_image_tensor,
bottleneck_tensor)
else:
train_bottlenecks, train_ground_truth = Bottleneck.get_random_cached_bottlenecks(
sess, image_lists, FLAGS.train_batch_size, 'training',
FLAGS.bottleneck_dir, FLAGS.image_dir, jpeg_data_tensor,
bottleneck_tensor)
# Feed the bottlenecks and ground truth into the graph, and run a training step.
summary, _ = sess.run(
[merged, train_step],
feed_dict={
bottleneck_input: train_bottlenecks,
ground_truth_input: train_ground_truth
})
# Add summaries to train writer
train_writer.add_summary(summary, i)
# Every so often, print out how well the graph is training.
is_last_step = (i + 1 == FLAGS.how_many_training_steps)
if (i % FLAGS.eval_step_interval) == 0 or is_last_step:
train_accuracy, cross_entropy_value = sess.run(
[evaluation_step, cross_entropy],
feed_dict={
bottleneck_input: train_bottlenecks,
ground_truth_input: train_ground_truth
})
print('%s: Step %d: Train accuracy = %.1f%%' %
(datetime.now(), i, train_accuracy * 100))
print('%s: Step %d: Cross entropy = %f' %
(datetime.now(), i, cross_entropy_value))
validation_bottlenecks, validation_ground_truth = (
Bottleneck.get_random_cached_bottlenecks(
sess, image_lists, FLAGS.validation_batch_size,
'validation', FLAGS.bottleneck_dir, FLAGS.image_dir,
jpeg_data_tensor, bottleneck_tensor))
summary, validation_accuracy = sess.run(
[merged, evaluation_step],
feed_dict={
bottleneck_input: validation_bottlenecks,
ground_truth_input: validation_ground_truth
})
# Add summaries to validation writer
validation_writer.add_summary(summary, i)
print('%s: Step %d: Validation accuracy = %.1f%%' %
(datetime.now(), i, validation_accuracy * 100))
# We've completed all our training, so run a final test evaluation on some new images we haven't used before.
test_bottlenecks, test_ground_truth = Bottleneck.get_random_cached_bottlenecks(
sess, image_lists, FLAGS.test_batch_size, 'testing',
FLAGS.bottleneck_dir, FLAGS.image_dir, jpeg_data_tensor,
bottleneck_tensor)
test_accuracy = sess.run(evaluation_step,
feed_dict={
bottleneck_input: test_bottlenecks,
ground_truth_input: test_ground_truth
})
print('Final test accuracy = %.1f%%' % (test_accuracy * 100))
# Write out the trained graph and labels with the weights stored as constants.
output_graph_def = graph_util.convert_variables_to_constants(
sess, graph.as_graph_def(), [FLAGS.final_tensor_name])
with gfile.FastGFile(FLAGS.output_graph, 'wb') as f:
f.write(output_graph_def.SerializeToString())
with gfile.FastGFile(FLAGS.output_labels, 'w') as f:
f.write('\n'.join(image_lists.keys()) + '\n')
def run_training():
"""Train fully_connected for a number of steps."""
# Get the sets of images and labels for training, validation, and
# test on fully_connected.
# Look at the folder structure, and create lists of all the images.
data_sets = regression_data.create_image_lists(FLAGS.image_dir, FLAGS.testing_percentage,
FLAGS.validation_percentage)
# Tell TensorFlow that the model will be built into the default Graph.
with tf.Graph().as_default():
# Generate placeholders for the images and labels.
images_placeholder, labels_placeholder = placeholder_inputs(
FLAGS.batch_size, name="input_images")
# Build a Graph that computes predictions from the inference model.
inference = regression.inference(images_placeholder,
FLAGS.hidden1,
FLAGS.hidden2)
# logits = fully_connected.inference(images_placeholder,
# FLAGS.hidden1,
# FLAGS.hidden2)
# softmax = tf.nn.softmax(logits, name="final_result")
# Add to the Graph the Ops for loss calculation.
loss = regression.loss(inference, labels_placeholder)
# Add to the Graph the Ops that calculate and apply gradients.
train_op = regression.training(loss, FLAGS.learning_rate)
# Add the Op to compare the logits to the labels during evaluation.
eval_correct = regression.evaluation(inference, labels_placeholder)
# Build the summary operation based on the TF collection of Summaries.
summary_op = tf.merge_all_summaries()
# Create a saver for writing training checkpoints.
saver = tf.train.Saver()
# Create a session for running Ops on the Graph.
sess = tf.Session()
# graph = sess.graph
# Run the Op to initialize the variables.
init = tf.initialize_all_variables()
sess.run(init)
# Instantiate a SummaryWriter to output summaries and the Graph.
summary_writer = tf.train.SummaryWriter(FLAGS.train_dir,
graph_def=sess.graph_def)
# And then after everything is built, start the training loop.
for step in xrange(FLAGS.max_steps):
start_time = time.time()
# Fill a feed dictionary with the actual set of images and labels
# for this particular training step.
feed_dict = fill_feed_dict(data_sets.train,
images_placeholder,
labels_placeholder)
# Run one step of the model. The return values are the activations
# from the `train_op` (which is discarded) and the `loss` Op. To
# inspect the values of your Ops or variables, you may include them
# in the list passed to sess.run() and the value tensors will be
# returned in the tuple from the call.
_, loss_value, inference_val = sess.run([train_op, loss, inference],
feed_dict=feed_dict)
duration = time.time() - start_time
# Write the summaries and print an overview fairly often.
if step % 10 == 0:
# Print status to stdout.
print('Step %d: loss = %.6f (%.3f sec)' % (step, loss_value, duration))
# Update the events file.
summary_str = sess.run(summary_op, feed_dict=feed_dict)
summary_writer.add_summary(summary_str, step)
# Save a checkpoint and evaluate the model periodically.
if (step + 1) % 1000 == 0 or (step + 1) == FLAGS.max_steps:
# saver.save(sess, FLAGS.train_dir, global_step=step)
# Evaluate against the training set.
# print('Training Data Eval:')
# do_eval(sess,
# eval_correct,
# images_placeholder,
# labels_placeholder,
# data_sets.train)
# Evaluate against the validation set.
# print('Validation Data Eval:')
# do_eval(sess,
# eval_correct,
# images_placeholder,
# labels_placeholder,
# data_sets.validation)
# Evaluate against the test set.
# print('Test Data Eval:')
# do_eval(sess,
# eval_correct,
# images_placeholder,
# labels_placeholder,
# data_sets.test)
# Write out the trained graph and labels with the weights stored as constants.
output_graph_def = graph_util.convert_variables_to_constants(
sess, sess.graph.as_graph_def(), ["final_result"])
with gfile.FastGFile("output_graph.pb", 'wb') as f:
f.write(output_graph_def.SerializeToString())
def train_and_freeze_graph():
#####################################################################################
# train a LMS model
# generate 100 samples with Y = W * X + b, where X, Y are 2-D column vectors,
# and W and b have following values:
# / 1.0 0.0 \ / 1.0 \
# W = | | , b = | |
# \ 0.0, 2.0 / \ 0.5 /
W_real = np.array([[1.0, 0.0], [0.0, 2.0]])
b_real = np.array([[1.0], [0.5]])
X_data = np.random.rand(2, 100).astype(np.float32)
Y_data = np.dot(W_real, X_data) + b_real
# Try to find values for W and b that compute y_data = W * x_data + b
W = tf.Variable(tf.random_uniform([2, 2], -1.0, 1.0), name = "W_learn")
b = tf.Variable(tf.zeros([2, 1]), name = "b_learn")
x = tf.placeholder(tf.float32, [2, None], name = "x_input")
y = tf.add(tf.matmul(W, x), b, name = "y_guess")
# Minimize the mean squared errors.
loss = tf.reduce_mean(tf.square(y - Y_data))
optimizer = tf.train.GradientDescentOptimizer(0.5)
grads = optimizer.compute_gradients(loss)
apply_grad_op = optimizer.apply_gradients(grads)
# Before starting, initialize the variables. We will 'run' this first.
init = tf.initialize_all_variables()
# Launch the graph.
sess = tf.Session()
sess.run(init)
for i in xrange(200):
_, loss_value = sess.run([apply_grad_op, loss], feed_dict={x: X_data})
print("loss %f" % loss_value)
## gradients has the same shape as it's corresponding variable
if i % 50 == 0:
print("=======================================")
print("type of grads: %s, size %d" % (type(grads), len(grads)))
print("type of grad : %s, type of var %s" % (type(grads[0][0]), type(grads[0][1])))
for grad, var in grads:
print("gradients of %s:" % var.name)
print(sess.run(grad, feed_dict={x: X_data}))
print("=======================================")
W_value, b_value = sess.run([W, b])
print("after training:")
print("W is:")
print(W_value)
print("b is:")
print(b_value)
print("x.name %s" % x.name)
print("y.name %s" % y.name)
#####################################################################################
#####################################################################################
# freeze and save the trained model for later restoring
# Write out the trained graph and labels with the weights stored as constants.
output_graph_def = graph_util.convert_variables_to_constants(
sess, sess.graph.as_graph_def(), ["x_input", "y_guess"])
with gfile.FastGFile('draft_graph.pb', 'wb') as f:
f.write(output_graph_def.SerializeToString())
def learn(data_folder,
experts,
learning_rate=.001,
train_ratio=.8,
validation_ratio=.1,
test_ratio=.1,
save_every=10,
batch_size=2048,
hidden_size=1024,
dropout=.5,
epochs=500,
print_every=1,
model_dir='.',
perceptron=False,
mem_ratio=.95):
assert train_ratio + validation_ratio + test_ratio == 1, 'Train/validation/test ratios must sum up to 1'
data = read_data(data_folder, train_ratio, validation_ratio, test_ratio)
model_name = (
'''transfer_classifier_moe_epochs_{}_batch_{}_ratios_{}_{}_{}_'''
'''learning_rate_{}'''.format(epochs, batch_size, train_ratio,
validation_ratio, test_ratio,
learning_rate))
if perceptron:
model_name = '{}_perceptron.pb'.format(model_name)
else:
model_name = '{}_dropout_{}_hidden_size_{}.pb'.format(
model_name, dropout, hidden_size)
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=mem_ratio)
with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess:
local_experts = {}
for model in os.listdir(experts):
print('Loading {}'.format(model))
load_graph(os.path.join(args.experts, model))
stripped = model[20:]
h5 = stripped[:stripped.find('_')] # MESSY.
local_experts[h5] = sess.graph.get_tensor_by_name(
'{}output:0'.format(h5))
data.train._X = np.hstack([
data.train.X,
np.vstack([
flow(sess, local_experts, x)
for x in chunks(data.train.X, batch_size)
])
])
data.validation._X = np.hstack(
[data.validation.X,
flow(sess, local_experts, data.validation.X)])
data.test._X = np.hstack(
[data.test.X, flow(sess, local_experts, data.test.X)])
x = tf.placeholder('float',
shape=[None, data.train.X_features],
name='input')
y_ = tf.placeholder('float',
shape=[None, data.train.Y_features],
name='target')
if perceptron:
W = weight_variable([data.train.X_features, data.train.Y_features],
name='weights')
b = bias_variable([data.train.Y_features], name='bias')
logits = tf.matmul(x, W) + b
else:
W_in = weight_variable([data.train.X_features, hidden_size],
name='weights_in')
b_in = bias_variable([hidden_size], name='bias_in')
hidden = tf.matmul(x, W_in) + b_in
relu = tf.nn.relu(hidden)
keep_prob = tf.placeholder_with_default([1.], shape=None)
hidden_dropout = tf.nn.dropout(relu, keep_prob)
W_out = weight_variable([hidden_size, data.train.Y_features],
name='weights_out')
b_out = bias_variable([data.train.Y_features], name='bias_out')
logits = tf.matmul(relu, W_out) + b_out
y = tf.nn.softmax(logits, name='output')
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits, y_)
train_step = tf.train.AdamOptimizer(learning_rate).minimize(
cross_entropy)
correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, 'float'))
sess.run(tf.initialize_all_variables())
last_epoch = 0
t_epoch = time.time()
while data.train.epoch <= epochs:
epoch = data.train.epoch
batch_x, batch_y = data.train.next_batch(batch_size)
t_start = time.time()
feed_dict = {
x: batch_x,
y_: batch_y
} if perceptron else {
x: batch_x,
y_: batch_y,
keep_prob: dropout
}
train_step.run(feed_dict=feed_dict)
t_end = time.time() - t_start
if epoch > last_epoch:
if epoch % print_every == 0:
train_accuracy = accuracy.eval(feed_dict={
x: batch_x,
y_: batch_y
})
validation_accuracy = accuracy.eval(feed_dict={
x: data.validation.X,
y_: data.validation.Y
})
print(
'''Epoch {} train accuracy: {}, validation accuracy: {}. '''
'''{} states/sec, {} secs/epoch.'''.format(
epoch, train_accuracy, validation_accuracy,
batch_size / t_end,
time.time() - t_epoch))
if epoch % save_every == 0 or epoch == epochs:
output_graph_def = graph_util.convert_variables_to_constants(
sess, sess.graph.as_graph_def(), ['input', 'output'])
with gfile.FastGFile(os.path.join(model_dir, model_name),
'w') as f:
f.write(output_graph_def.SerializeToString())
t_epoch = time.time()
last_epoch = epoch
print('Trained model saved to {}'.format(
os.path.join(model_dir, model_name)))
if test_ratio > 0:
test_accuracy = accuracy.eval(feed_dict={
x: data.test.X,
y_: data.test.Y
})
print('Evaluation on testing data: {}'.format(test_accuracy))
def learn(
train_states,
test_states,
model,
learning_rate=0.0001,
save_every=10,
batch_size=2048,
hidden_size=2048,
dropout=0.5,
epochs=500,
print_every=1,
model_dir=".",
perceptron=False,
mem_ratio=0.95,
):
data = read_data(train_states, test_states)
model_name = """trust_classifier_epochs_{}_batch_{}_learning_rate_{}""".format(epochs, batch_size, learning_rate)
if perceptron:
model_name = "{}_perceptron.pb".format(model_name)
else:
model_name = "{}_dropout_{}_hidden_size_{}.pb".format(model_name, dropout, hidden_size)
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=mem_ratio)
with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess:
load_graph(model)
transfer_predictor = sess.graph.get_tensor_by_name("output:0")
# Evaluate the model on the training and test data, for training and testing.
data.train._X = np.vstack(
[sess.run(transfer_predictor, {"input:0": chunk}) for chunk in chunks(data.train.X, 10)]
)
answer = tf.equal(tf.argmax(data.train.X, 1), tf.argmax(data.train.Y, 1))
data.train._Y = tf.one_hot(tf.to_int32(answer), depth=2).eval()
data.test._X = sess.run(transfer_predictor, {"input:0": data.test.X})
answer = tf.equal(tf.argmax(data.test.X, 1), tf.argmax(data.test.Y, 1))
data.test._Y = tf.one_hot(tf.to_int32(answer), depth=2).eval()
x = tf.placeholder("float", shape=[None, data.train.X_features], name="input_b")
y_ = tf.placeholder("float", shape=[None, data.train.Y_features], name="target")
if perceptron:
W = weight_variable([data.train.X_features, data.train.Y_features], name="weights")
b = bias_variable([data.train.Y_features], name="bias")
logits = tf.matmul(x, W) + b
else:
W_in = weight_variable([data.train.X_features, hidden_size], name="weights_in")
b_in = bias_variable([hidden_size], name="bias_in")
hidden = tf.matmul(x, W_in) + b_in
relu = tf.nn.relu(hidden)
keep_prob = tf.placeholder_with_default([1.0], shape=None)
hidden_dropout = tf.nn.dropout(relu, keep_prob)
W_out = weight_variable([hidden_size, data.train.Y_features], name="weights_out")
b_out = bias_variable([data.train.Y_features], name="bias_out")
logits = tf.matmul(relu, W_out) + b_out
# Loss & train
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits, y_)
train_step = tf.train.AdamOptimizer(learning_rate).minimize(cross_entropy)
# Evaluation
y = tf.nn.softmax(logits, name="output_b")
correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
sess.run(tf.initialize_all_variables())
last_epoch = 0
t_epoch = time.time()
while data.train.epoch <= epochs:
epoch = data.train.epoch
batch_x, batch_y = data.train.next_batch(batch_size)
t_start = time.time()
feed_dict = {x: batch_x, y_: batch_y} if perceptron else {x: batch_x, y_: batch_y, keep_prob: dropout}
train_step.run(feed_dict=feed_dict)
t_end = time.time() - t_start
if epoch > last_epoch:
if epoch % print_every == 0:
train_accuracy_mean = accuracy.eval(feed_dict={x: batch_x, y_: batch_y})
validation_accuracy_mean = accuracy.eval(feed_dict={x: data.test.X, y_: data.test.Y})
print(
"""Epoch {} train accuracy: {}, test accuracy: {}. """
"""{} states/sec, {} secs/epoch.""".format(
epoch,
train_accuracy_mean,
validation_accuracy_mean,
batch_size / t_end,
time.time() - t_epoch,
)
)
if epoch % save_every == 0 or epoch == epochs:
output_graph_def = graph_util.convert_variables_to_constants(
sess, sess.graph.as_graph_def(), ["input_b", "output_b"]
)
with gfile.FastGFile(os.path.join(model_dir, model_name), "w") as f:
f.write(output_graph_def.SerializeToString())
t_epoch = time.time()
last_epoch = epoch
print("Trained model saved to {}".format(os.path.join(model_dir, model_name)))
def learn(train_states,
test_states,
model,
learning_rate=.0001,
save_every=10,
batch_size=2048,
hidden_size=2048,
dropout=.5,
epochs=500,
print_every=1,
model_dir='.',
perceptron=False,
mem_ratio=.95):
data = read_data(train_states, test_states)
model_name = (
'''trust_classifier_epochs_{}_batch_{}_learning_rate_{}'''.format(
epochs, batch_size, learning_rate))
if perceptron:
model_name = '{}_perceptron.pb'.format(model_name)
else:
model_name = '{}_dropout_{}_hidden_size_{}.pb'.format(
model_name, dropout, hidden_size)
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=mem_ratio)
with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess:
load_graph(model)
transfer_predictor = sess.graph.get_tensor_by_name('output:0')
# Evaluate the model on the training and test data, for training and testing.
data.train._X = np.vstack([
sess.run(transfer_predictor, {'input:0': chunk})
for chunk in chunks(data.train.X, 10)
])
answer = tf.equal(tf.argmax(data.train.X, 1),
tf.argmax(data.train.Y, 1))
data.train._Y = tf.one_hot(tf.to_int32(answer), depth=2).eval()
data.test._X = sess.run(transfer_predictor, {'input:0': data.test.X})
answer = tf.equal(tf.argmax(data.test.X, 1), tf.argmax(data.test.Y, 1))
data.test._Y = tf.one_hot(tf.to_int32(answer), depth=2).eval()
x = tf.placeholder('float',
shape=[None, data.train.X_features],
name='input_b')
y_ = tf.placeholder('float',
shape=[None, data.train.Y_features],
name='target')
if perceptron:
W = weight_variable([data.train.X_features, data.train.Y_features],
name='weights')
b = bias_variable([data.train.Y_features], name='bias')
logits = tf.matmul(x, W) + b
else:
W_in = weight_variable([data.train.X_features, hidden_size],
name='weights_in')
b_in = bias_variable([hidden_size], name='bias_in')
hidden = tf.matmul(x, W_in) + b_in
relu = tf.nn.relu(hidden)
keep_prob = tf.placeholder_with_default([1.], shape=None)
hidden_dropout = tf.nn.dropout(relu, keep_prob)
W_out = weight_variable([hidden_size, data.train.Y_features],
name='weights_out')
b_out = bias_variable([data.train.Y_features], name='bias_out')
logits = tf.matmul(relu, W_out) + b_out
# Loss & train
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits, y_)
train_step = tf.train.AdamOptimizer(learning_rate).minimize(
cross_entropy)
# Evaluation
y = tf.nn.softmax(logits, name='output_b')
correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, 'float'))
sess.run(tf.initialize_all_variables())
last_epoch = 0
t_epoch = time.time()
while data.train.epoch <= epochs:
epoch = data.train.epoch
batch_x, batch_y = data.train.next_batch(batch_size)
t_start = time.time()
feed_dict = {
x: batch_x,
y_: batch_y
} if perceptron else {
x: batch_x,
y_: batch_y,
keep_prob: dropout
}
train_step.run(feed_dict=feed_dict)
t_end = time.time() - t_start
if epoch > last_epoch:
if epoch % print_every == 0:
train_accuracy_mean = accuracy.eval(feed_dict={
x: batch_x,
y_: batch_y
})
validation_accuracy_mean = accuracy.eval(feed_dict={
x: data.test.X,
y_: data.test.Y
})
print(
'''Epoch {} train accuracy: {}, test accuracy: {}. '''
'''{} states/sec, {} secs/epoch.'''.format(
epoch, train_accuracy_mean,
validation_accuracy_mean, batch_size / t_end,
time.time() - t_epoch))
if epoch % save_every == 0 or epoch == epochs:
output_graph_def = graph_util.convert_variables_to_constants(
sess, sess.graph.as_graph_def(),
['input_b', 'output_b'])
with gfile.FastGFile(os.path.join(model_dir, model_name),
'w') as f:
f.write(output_graph_def.SerializeToString())
t_epoch = time.time()
last_epoch = epoch
print('Trained model saved to {}'.format(
os.path.join(model_dir, model_name)))
def learn(train_states, test_states, learning_rate, save_every, batch_size, hidden_size, dropout, epochs,
print_every, model_dir, perceptron, dense, lenet, filter_width, depth, mem_ratio,
q_size, use_dask, in_memory, dask_chunksize):
data = read_data(train_states, test_states, use_dask, in_memory, dask_chunksize)
model_name = ('''transfer_classifier_epochs_{}_batch_{}_learning_rate_{}'''.format(
epochs, batch_size, learning_rate))
if perceptron:
model_name = '{}_perceptron.pb'.format(model_name)
if dense:
model_name = '{}_dense_dropout_{}_hidden_size_{}.pb'.format(model_name, dropout, hidden_size)
if lenet:
model_name = '{}_lenet_dropout_{}_hidden_size_{}.pb'.format(model_name, dropout, hidden_size)
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=mem_ratio)
with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess:
q_x_in = tf.placeholder(tf.float32, shape=[batch_size, data.train.X_features])
q_y_in = tf.placeholder(tf.int32, shape=[batch_size])
y_real = tf.placeholder(tf.int32, shape=[None])
keep_prob = tf.placeholder_with_default([1.], shape=None)
q = tf.FIFOQueue(q_size, [tf.float32, tf.int32],
shapes=[ q_x_in.get_shape(), q_y_in.get_shape()])
enqueue_op = q.enqueue([q_x_in, q_y_in])
q_x_out, q_y_out = q.dequeue()
x = tf.placeholder_with_default(q_x_out, shape=[None, data.train.X_features], name='input')
y_ = tf.placeholder_with_default(q_y_out, shape=[None])
if perceptron:
W = weight_variable([data.train.X_features, data.train.Y_features])
b = bias_variable([data.train.Y_features])
logits = tf.matmul(x, W) + b
if dense:
W_in = weight_variable([data.train.X_features, hidden_size])
b_in = bias_variable([hidden_size])
hidden = tf.matmul(x, W_in) + b_in
relu = tf.nn.relu(hidden)
hidden_dropout = tf.nn.dropout(relu, keep_prob)
W_out = weight_variable([hidden_size,data.train.Y_features])
b_out = bias_variable([data.train.Y_features])
logits = tf.matmul(hidden_dropout, W_out) + b_out
if lenet:
w1 = weight_variable([1, filter_width, 1, depth])
b1 = bias_variable([depth])
x_4d = tf.expand_dims(tf.expand_dims(x,1),-1) # Singleton dimension height, out_channel
conv = tf.nn.conv2d(x_4d, w1, strides=[1,1,1,1], padding='SAME')
relu = tf.nn.relu(tf.nn.bias_add(conv, b1))
pool = tf.nn.max_pool(relu, ksize=[1, 1, 2, 1], strides=[1, 1, 2, 1], padding='SAME')
w2 = weight_variable([1, filter_width, depth, depth*2])
b2 = bias_variable([depth*2])
conv = tf.nn.conv2d(pool, w2, strides=[1,1,1,1], padding='SAME')
relu = tf.nn.relu(tf.nn.bias_add(conv, b2))
pool = tf.nn.max_pool(relu, ksize=[1, 1, 2, 1], strides=[1, 1, 2, 1], padding='SAME')
pool_shape = tf.shape(pool)
reshape = tf.reshape(pool,
[pool_shape[0], pool_shape[1] * pool_shape[2] * pool_shape[3] ])
w3 = weight_variable([ (data.train.X_features/4)*2*depth, hidden_size])
b3 = bias_variable([hidden_size])
hidden = tf.matmul(reshape, w3) + b3
relu = tf.nn.relu(hidden)
hidden_dropout = tf.nn.dropout(relu, keep_prob)
w4 = weight_variable([hidden_size, data.train.Y_features])
b4 = bias_variable([data.train.Y_features])
logits = tf.matmul(hidden_dropout, w4) + b4
# Loss & train
cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits, y_)
train_step = tf.train.AdamOptimizer(learning_rate).minimize(cross_entropy)
# Evaluation
y = tf.nn.softmax(logits)
train_correct_prediction = tf.equal(tf.to_int32(tf.argmax(y, 1)), y_,)
train_accuracy = tf.reduce_mean(tf.cast(train_correct_prediction, tf.float32))
# Applying convolutional filter to put subcategories into original categories for testing
stride = [1,1,1,1]
_filter = tf.constant(data.output_filter, dtype=tf.float32, shape=data.output_filter.shape)
conv_in = tf.expand_dims(y, 0)
conv_in = tf.expand_dims(conv_in,-1)
conv_out = tf.nn.conv2d(conv_in, _filter, stride, 'VALID') # We don't want zero padding.
back = tf.squeeze(conv_out, squeeze_dims=[0,2], name='output')
test_correct_prediction = tf.equal(tf.to_int32(tf.argmax(back, 1)), y_real)
test_accuracy = tf.reduce_mean(tf.cast(test_correct_prediction, tf.float32))
sess.run(tf.initialize_all_variables())
def load_data():
try:
while True:
next_x, next_y = data.train.next_batch(batch_size)
if next_x.shape[0] == batch_size:
sess.run(enqueue_op, feed_dict={q_x_in: next_x, q_y_in: next_y})
except Exception as error:
print(error)
print('Stopped streaming of data.')
data_thread = threading.Thread(target=load_data)
data_thread.daemon = True
data_thread.start()
last_epoch = 0
epoch = 0
t_epoch = time.time()
t_end = []
i = 0
while epoch <= epochs:
t_start = time.time()
sess.run(train_step, feed_dict={keep_prob: dropout})
t_end.append(time.time() - t_start)
if epoch > last_epoch:
if epoch % print_every == 0:
batch_x, batch_y = data.train.next_batch(batch_size)
train_accuracy_mean = train_accuracy.eval(feed_dict={
x: batch_x,
y_: batch_y })
validation_accuracy_mean = np.mean([ test_accuracy.eval(feed_dict={x: t_x, y_real: t_y })
for t_x, t_y in zip(chunks(data.test.X, batch_size),
chunks(data.test.Y, batch_size)) ])
print('''Epoch {} train accuracy: {}, test accuracy: {}. '''
'''{} states/sec on average, {} secs/epoch.'''.format(epoch, pf(train_accuracy_mean),
pf(validation_accuracy_mean),
pf(batch_size/np.mean(t_end)),
pf(time.time() - t_epoch)))
if epoch % save_every == 0 or epoch == epochs:
output_graph_def = graph_util.convert_variables_to_constants(
sess, sess.graph.as_graph_def(), ['input', 'output'])
with gfile.FastGFile(os.path.join(model_dir, model_name), 'w') as f:
f.write(output_graph_def.SerializeToString())
t_epoch = time.time()
t_end = []
last_epoch = epoch
i += 1
if batch_size*i > data.train.X_len:
epoch += 1
i = 0
q.close(cancel_pending_enqueues=True)
print('Trained model saved to {}'.format(os.path.join(model_dir, model_name)))
from tensorflow.python.client import graph_util
v1 = tf.Variable(tf.constant(1.0, shape=[1]), name = "v1")
v2 = tf.Variable(tf.constant(2.0, shape=[1]), name = "v2")
result = v1 + v2
print result
v3 = tf.Variable(tf.constant(3.0, shape=[1]), name = "v1")
v4 = tf.Variable(tf.constant(4.0, shape=[1]), name = "v2")
result = v3 + v4
print result
init_op = tf.initialize_all_variables()
with tf.Session() as sess:
sess.run(init_op)
graph_def = tf.get_default_graph().as_graph_def()
output_graph_def = graph_util.convert_variables_to_constants(sess, graph_def, ['add_1'])
with tf.gfile.GFile("../../datasets/combined_model.pb", "wb") as f:
f.write(output_graph_def.SerializeToString())
from tensorflow.python.platform import gfile
with tf.Session() as sess:
model_filename = "../../datasets/combined_model.pb"
with gfile.FastGFile(model_filename, 'rb1') as f:
graph_def = tf.GraphDef()
graph_def.ParseFromString(f.read())
result = tf.import_graph_def(graph_def, return_elements=["add_1:0"])
print sess.run(result)
def main(_):
# Set up the pre-trained graph.
maybe_download_and_extract()
graph, bottleneck_tensor, jpeg_data_tensor, resized_image_tensor = (
create_inception_graph())
# Get drviers list
imagesTodrivers=get_driver_data()
# Look at the folder structure, and create lists of all the images.
image_lists = create_image_lists(FLAGS.image_dir, FLAGS.testing_percentage,
FLAGS.validation_percentage, imagesTodrivers, driversToPutInTesting, driversToPutInCV, usingProcessed)
class_count = len(image_lists.keys())
if class_count == 0:
print('No valid folders of images found at ' + FLAGS.image_dir)
return -1
if class_count == 1:
print('Only one valid folder of images found at ' + FLAGS.image_dir +
' - multiple classes are needed for classification.')
return -1
# See if the command-line flags mean we're applying any distortions.
do_distort_images = should_distort_images(
FLAGS.flip_left_right, FLAGS.random_crop, FLAGS.random_scale,
FLAGS.random_brightness)
sess = tf.Session()
if do_distort_images:
# We will be applying distortions, so setup the operations we'll need.
distorted_jpeg_data_tensor, distorted_image_tensor = add_input_distortions(
FLAGS.flip_left_right, FLAGS.random_crop, FLAGS.random_scale,
FLAGS.random_brightness)
else:
# We'll make sure we've calculated the 'bottleneck' image summaries and
# cached them on disk.
cache_bottlenecks(sess, image_lists, FLAGS.image_dir, FLAGS.bottleneck_dir,
jpeg_data_tensor, bottleneck_tensor)
# Add the new layer that we'll be training.
(train_step, cross_entropy, bottleneck_input, ground_truth_input,
final_tensor) = add_final_training_ops(len(image_lists.keys()),
FLAGS.final_tensor_name,
bottleneck_tensor)
# Set up all our weights to their initial default values.
init = tf.initialize_all_variables()
sess.run(init)
# Create the operations we need to evaluate the accuracy of our new layer.
evaluation_step = add_evaluation_step(final_tensor, ground_truth_input)
# Run the training for as many cycles as requested on the command line.
for i in range(FLAGS.how_many_training_steps):
# Get a catch of input bottleneck values, either calculated fresh every time
# with distortions applied, or from the cache stored on disk.
if do_distort_images:
train_bottlenecks, train_ground_truth = get_random_distorted_bottlenecks(
sess, image_lists, FLAGS.train_batch_size, 'training',
FLAGS.image_dir, distorted_jpeg_data_tensor,
distorted_image_tensor, resized_image_tensor, bottleneck_tensor)
else:
train_bottlenecks, train_ground_truth = get_random_cached_bottlenecks(
sess, image_lists, FLAGS.train_batch_size, 'training',
FLAGS.bottleneck_dir, FLAGS.image_dir, jpeg_data_tensor,
bottleneck_tensor)
# Feed the bottlenecks and ground truth into the graph, and run a training
# step.
sess.run(train_step,
feed_dict={bottleneck_input: train_bottlenecks,
ground_truth_input: train_ground_truth})
# Every so often, print out how well the graph is training.
is_last_step = (i + 1 == FLAGS.how_many_training_steps)
if (i % FLAGS.eval_step_interval) == 0 or is_last_step:
train_accuracy, cross_entropy_value = sess.run(
[evaluation_step, cross_entropy],
feed_dict={bottleneck_input: train_bottlenecks,
ground_truth_input: train_ground_truth})
print('%s: Step %d: Train accuracy = %.1f%%' % (datetime.now(), i,
train_accuracy * 100))
print('%s: Step %d: Cross entropy = %f' % (datetime.now(), i,
cross_entropy_value))
validation_bottlenecks, validation_ground_truth = (
get_random_cached_bottlenecks(
sess, image_lists, FLAGS.validation_batch_size, 'validation',
FLAGS.bottleneck_dir, FLAGS.image_dir, jpeg_data_tensor,
bottleneck_tensor))
validation_accuracy = sess.run(
evaluation_step,
feed_dict={bottleneck_input: validation_bottlenecks,
ground_truth_input: validation_ground_truth})
print('%s: Step %d: Validation accuracy = %.1f%%' %
(datetime.now(), i, validation_accuracy * 100))
# We've completed all our training, so run a final test evaluation on
# some new images we haven't used before.
test_bottlenecks, test_ground_truth = get_random_cached_bottlenecks(
sess, image_lists, FLAGS.test_batch_size, 'testing',
FLAGS.bottleneck_dir, FLAGS.image_dir, jpeg_data_tensor,
bottleneck_tensor)
test_accuracy = sess.run(
evaluation_step,
feed_dict={bottleneck_input: test_bottlenecks,
ground_truth_input: test_ground_truth})
print('Final test accuracy = %.1f%%' % (test_accuracy * 100))
# Write out the trained graph and labels with the weights stored as constants.
output_graph_def = graph_util.convert_variables_to_constants(
sess, graph.as_graph_def(), [FLAGS.final_tensor_name])
with gfile.FastGFile(FLAGS.output_graph, 'wb') as f:
f.write(output_graph_def.SerializeToString())
with gfile.FastGFile(FLAGS.output_labels, 'w') as f:
f.write('\n'.join(image_lists.keys()) + '\n')
def learn(data_folder, experts, learning_rate=.001, train_ratio=.8, validation_ratio=.1, test_ratio=.1, save_every=10, batch_size=2048, hidden_size=1024, dropout=.5, epochs=500, print_every=1, model_dir='.', perceptron=False, mem_ratio=.95):
assert train_ratio + validation_ratio + test_ratio == 1, 'Train/validation/test ratios must sum up to 1'
data = read_data(data_folder, train_ratio, validation_ratio, test_ratio)
model_name = ('''transfer_classifier_moe_epochs_{}_batch_{}_ratios_{}_{}_{}_'''
'''learning_rate_{}'''.format(
epochs, batch_size,
train_ratio, validation_ratio,
test_ratio, learning_rate))
if perceptron:
model_name = '{}_perceptron.pb'.format(model_name)
else:
model_name = '{}_dropout_{}_hidden_size_{}.pb'.format(model_name, dropout, hidden_size)
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=mem_ratio)
with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess:
local_experts = {}
for model in os.listdir(experts):
print('Loading {}'.format(model))
load_graph(os.path.join(args.experts, model))
stripped = model[20:]
h5 = stripped[:stripped.find('_')]# MESSY.
local_experts[h5] = sess.graph.get_tensor_by_name('{}output:0'.format(h5))
data.train._X = np.hstack([ data.train.X, np.vstack([ flow(sess, local_experts, x) for x in chunks(data.train.X, batch_size) ]) ])
data.validation._X = np.hstack([ data.validation.X, flow(sess, local_experts, data.validation.X) ])
data.test._X = np.hstack([ data.test.X, flow(sess, local_experts, data.test.X) ])
x = tf.placeholder('float', shape=[None, data.train.X_features], name='input')
y_ = tf.placeholder('float', shape=[None, data.train.Y_features], name='target')
if perceptron:
W = weight_variable([data.train.X_features, data.train.Y_features], name='weights')
b = bias_variable([data.train.Y_features], name='bias')
logits = tf.matmul(x,W) + b
else:
W_in = weight_variable([data.train.X_features, hidden_size], name='weights_in')
b_in = bias_variable([hidden_size], name='bias_in')
hidden = tf.matmul(x,W_in) + b_in
relu = tf.nn.relu(hidden)
keep_prob = tf.placeholder_with_default([1.], shape=None)
hidden_dropout = tf.nn.dropout(relu, keep_prob)
W_out = weight_variable([hidden_size,data.train.Y_features], name='weights_out')
b_out = bias_variable([data.train.Y_features], name='bias_out')
logits = tf.matmul(relu,W_out) + b_out
y = tf.nn.softmax(logits, name='output')
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits, y_)
train_step = tf.train.AdamOptimizer(learning_rate).minimize(cross_entropy)
correct_prediction = tf.equal(tf.argmax(y,1), tf.argmax(y_,1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, 'float'))
sess.run(tf.initialize_all_variables())
last_epoch = 0
t_epoch = time.time()
while data.train.epoch <= epochs:
epoch = data.train.epoch
batch_x, batch_y = data.train.next_batch(batch_size)
t_start = time.time()
feed_dict = {x: batch_x, y_: batch_y } if perceptron else {x: batch_x, y_: batch_y, keep_prob: dropout}
train_step.run(feed_dict=feed_dict)
t_end = time.time() - t_start
if epoch > last_epoch:
if epoch % print_every == 0:
train_accuracy = accuracy.eval(feed_dict={
x: batch_x,
y_: batch_y })
validation_accuracy = accuracy.eval(feed_dict={
x: data.validation.X,
y_: data.validation.Y })
print('''Epoch {} train accuracy: {}, validation accuracy: {}. '''
'''{} states/sec, {} secs/epoch.'''.format(epoch, train_accuracy,
validation_accuracy, batch_size/t_end,
time.time() - t_epoch))
if epoch % save_every == 0 or epoch == epochs:
output_graph_def = graph_util.convert_variables_to_constants(
sess, sess.graph.as_graph_def(), ['input', 'output'])
with gfile.FastGFile(os.path.join(model_dir, model_name), 'w') as f:
f.write(output_graph_def.SerializeToString())
t_epoch = time.time()
last_epoch = epoch
print('Trained model saved to {}'.format(os.path.join(model_dir, model_name)))
if test_ratio > 0:
test_accuracy = accuracy.eval(feed_dict={x: data.test.X, y_: data.test.Y })
print('Evaluation on testing data: {}'.format(test_accuracy))
