def freeze_graph(input_graph, input_saver, input_binary, input_checkpoint, output_node_names, restore_op_name, filename_tensor_name, output_graph, clear_devices, initializer_nodes): """Converts all variables in a graph and checkpoint into constants.""" if not tf.gfile.Exists(input_graph): print("Input graph file '" + input_graph + "' does not exist!") return -1 if input_saver and not tf.gfile.Exists(input_saver): print("Input saver file '" + input_saver + "' does not exist!") return -1 if not tf.gfile.Glob(input_checkpoint): print("Input checkpoint '" + input_checkpoint + "' doesn't exist!") return -1 if not output_node_names: print("You need to supply the name of a node to --output_node_names.") return -1 input_graph_def = tf.GraphDef() mode = "rb" if input_binary else "r" with tf.gfile.FastGFile(input_graph, mode) as f: if input_binary: input_graph_def.ParseFromString(f.read()) else: text_format.Merge(f.read(), input_graph_def) # Remove all the explicit device specifications for this node. This helps to # make the graph more portable. if clear_devices: for node in input_graph_def.node: node.device = "" _ = tf.import_graph_def(input_graph_def, name="") with tf.Session() as sess: if input_saver: with tf.gfile.FastGFile(input_saver, mode) as f: saver_def = tf.train.SaverDef() if input_binary: saver_def.ParseFromString(f.read()) else: text_format.Merge(f.read(), saver_def) saver = tf.train.Saver(saver_def=saver_def) saver.restore(sess, input_checkpoint) else: sess.run([restore_op_name], {filename_tensor_name: input_checkpoint}) if initializer_nodes: sess.run(initializer_nodes) output_graph_def = graph_util.convert_variables_to_constants( sess, input_graph_def, output_node_names.split(",")) with tf.gfile.GFile(output_graph, "wb") as f: f.write(output_graph_def.SerializeToString()) print("%d ops in the final graph." % len(output_graph_def.node))
def testConvertVariablesToConsts(self): with tf.Graph().as_default(): variable_node = tf.Variable(1.0, name="variable_node") _ = tf.Variable(1.0, name="unused_variable_node") output_node = tf.mul(variable_node, 2.0, name="output_node") with tf.Session() as sess: init = tf.initialize_variables([variable_node]) sess.run(init) output = sess.run(output_node) self.assertNear(2.0, output, 0.00001) variable_graph_def = sess.graph.as_graph_def() # First get the constant_graph_def when variable_names_whitelist is set, # note that if variable_names_whitelist is not set an error will be # thrown because unused_variable_node is not initialized. constant_graph_def = graph_util.convert_variables_to_constants( sess, variable_graph_def, ["output_node"], variable_names_whitelist=set(["variable_node"])) # Then initialize the unused variable, and get another # constant_graph_def when variable_names_whitelist is not set. sess.run(tf.initialize_all_variables()) constant_graph_def_without_variable_whitelist = ( graph_util.convert_variables_to_constants( sess, variable_graph_def, ["output_node"])) # The unused variable should be cleared so the two graphs should be # equivalent. self.assertEqual( str(constant_graph_def), str(constant_graph_def_without_variable_whitelist)) # Now we make sure the variable is now a constant, and that the graph still # produces the expected result. with tf.Graph().as_default(): _ = tf.import_graph_def(constant_graph_def, name="") self.assertEqual(4, len(constant_graph_def.node)) for node in constant_graph_def.node: self.assertNotEqual("Variable", node.op) with tf.Session() as sess: output_node = sess.graph.get_tensor_by_name("output_node:0") output = sess.run(output_node) self.assertNear(2.0, output, 0.00001)
def freeze_graph(input_graph, input_saver, input_binary, input_checkpoint, output_node_names, restore_op_name, filename_tensor_name, output_graph, clear_devices, initializer_nodes): """Converts all variables in a graph and checkpoint into constants.""" if not tf.gfile.Exists(input_graph): print("Input graph file '" + input_graph + "' does not exist!") return -1 if input_saver and not tf.gfile.Exists(input_saver): print("Input saver file '" + input_saver + "' does not exist!") return -1 if not tf.gfile.Glob(input_checkpoint): print("Input checkpoint '" + input_checkpoint + "' doesn't exist!") return -1 if not output_node_names: print("You need to supply the name of a node to --output_node_names.") return -1 input_graph_def = tf.GraphDef() mode = "rb" if input_binary else "r" with tf.gfile.FastGFile(input_graph, mode) as f: if input_binary: input_graph_def.ParseFromString(f.read()) else: text_format.Merge(f.read(), input_graph_def) # Remove all the explicit device specifications for this node. This helps to # make the graph more portable. if clear_devices: for node in input_graph_def.node: node.device = "" _ = tf.import_graph_def(input_graph_def, name="") with tf.Session() as sess: if input_saver: with tf.gfile.FastGFile(input_saver, mode) as f: saver_def = tf.train.SaverDef() if input_binary: saver_def.ParseFromString(f.read()) else: text_format.Merge(f.read(), saver_def) saver = tf.train.Saver(saver_def=saver_def) saver.restore(sess, input_checkpoint) else: sess.run([restore_op_name], {filename_tensor_name: input_checkpoint}) if initializer_nodes: sess.run(initializer_nodes) output_graph_def = graph_util.convert_variables_to_constants( sess, input_graph_def, output_node_names.split(",")) with tf.gfile.GFile(output_graph, "wb") as f: f.write(output_graph_def.SerializeToString()) print("%d ops in the final graph." % len(output_graph_def.node))
def testConvertVariablesToConsts(self): with tf.Graph().as_default(): variable_node = tf.Variable(1.0, name="variable_node") _ = tf.Variable(1.0, name="unused_variable_node") output_node = tf.mul(variable_node, 2.0, name="output_node") with tf.Session() as sess: init = tf.initialize_variables([variable_node]) sess.run(init) output = sess.run(output_node) self.assertNear(2.0, output, 0.00001) variable_graph_def = sess.graph.as_graph_def() # First get the constant_graph_def when variable_names_whitelist is set, # note that if variable_names_whitelist is not set an error will be # thrown because unused_variable_node is not initialized. constant_graph_def = graph_util.convert_variables_to_constants( sess, variable_graph_def, ["output_node"], variable_names_whitelist=set(["variable_node"])) # Then initialize the unused variable, and get another # constant_graph_def when variable_names_whitelist is not set. sess.run(tf.initialize_all_variables()) constant_graph_def_without_variable_whitelist = ( graph_util.convert_variables_to_constants( sess, variable_graph_def, ["output_node"])) # The unused variable should be cleared so the two graphs should be # equivalent. self.assertEqual(str(constant_graph_def), str(constant_graph_def_without_variable_whitelist)) # Now we make sure the variable is now a constant, and that the graph still # produces the expected result. with tf.Graph().as_default(): _ = tf.import_graph_def(constant_graph_def, name="") self.assertEqual(4, len(constant_graph_def.node)) for node in constant_graph_def.node: self.assertNotEqual("Variable", node.op) with tf.Session() as sess: output_node = sess.graph.get_tensor_by_name("output_node:0") output = sess.run(output_node) self.assertNear(2.0, output, 0.00001)
def export(): with tf.Graph().as_default() as graph: with tf.Session() as sess: model = InceptionModel(ds) model.build_graph(for_training=False) model.load_last_checkpoint(sess) output_file = os.path.join(MODEL_DIR, str(model) + ".pb") graph_def = graph.as_graph_def() graph_def = convert_variables_to_constants(sess, graph_def, ["results/predictions"]) with gfile.FastGFile(output_file, 'wb') as f: f.write(graph_def.SerializeToString())
def freeze_graph(model_folder): # We retrieve our checkpoint fullpath checkpoint = tf.train.get_checkpoint_state(model_folder) input_checkpoint = checkpoint.model_checkpoint_path # We precise the file fullname of our freezed graph absolute_model_folder = "/".join(input_checkpoint.split('/')[:-1]) output_graph = absolute_model_folder + "/frozen_model.pb" # Before exporting our graph, we need to precise what is our output node # This is how TF decides what part of the Graph he has to keep and what part it can dump # NOTE: this variable is plural, because you can have multiple output nodes output_node_names = "Accuracy/predictions" # We clear devices to allow TensorFlow to control on which device it will load operations clear_devices = True # We import the meta graph and retrieve a Saver saver = tf.train.import_meta_graph(input_checkpoint + '.meta', clear_devices=clear_devices) # We retrieve the protobuf graph definition graph = tf.get_default_graph() input_graph_def = graph.as_graph_def() # We start a session and restore the graph weights with tf.Session() as sess: saver.restore(sess, input_checkpoint) # We use a built-in TF helper to export variables to constants output_graph_def = graph_util.convert_variables_to_constants( sess, # The session is used to retrieve the weights input_graph_def, # The graph_def is used to retrieve the nodes output_node_names.split( "," ) # The output node names are used to select the usefull nodes ) # Finally we serialize and dump the output graph to the filesystem with tf.gfile.GFile(output_graph, "wb") as f: f.write(output_graph_def.SerializeToString()) print("%d ops in the final graph." % len(output_graph_def.node))
def testConvertVariablesToConsts(self): with tf.Graph().as_default(): variable_node = tf.Variable(1.0, name="variable_node") output_node = tf.mul(variable_node, 2.0, name="output_node") with tf.Session() as sess: init = tf.initialize_all_variables() sess.run(init) output = sess.run(output_node) self.assertNear(2.0, output, 0.00001) variable_graph_def = sess.graph.as_graph_def() constant_graph_def = graph_util.convert_variables_to_constants( sess, variable_graph_def, ["output_node"]) # Now we make sure the variable is now a constant, and that the graph still # produces the expected result. with tf.Graph().as_default(): _ = tf.import_graph_def(constant_graph_def, name="") self.assertEqual(4, len(constant_graph_def.node)) for node in constant_graph_def.node: self.assertNotEqual("Variable", node.op) with tf.Session() as sess: output_node = sess.graph.get_tensor_by_name("output_node:0") output = sess.run(output_node) self.assertNear(2.0, output, 0.00001)
def testConvertVariablesToConsts(self): with tf.Graph().as_default(): variable_node = tf.Variable(1.0, name="variable_node") output_node = tf.mul(variable_node, 2.0, name="output_node") with tf.Session() as sess: init = tf.initialize_all_variables() sess.run(init) output = sess.run(output_node) self.assertNear(2.0, output, 0.00001) variable_graph_def = sess.graph.as_graph_def() constant_graph_def = graph_util.convert_variables_to_constants( sess, variable_graph_def, ["output_node"]) # Now we make sure the variable is now a constant, and that the graph still # produces the expected result. with tf.Graph().as_default(): _ = tf.import_graph_def(constant_graph_def, name="") self.assertEqual(4, len(constant_graph_def.node)) for node in constant_graph_def.node: self.assertNotEqual("Variable", node.op) with tf.Session() as sess: output_node = sess.graph.get_tensor_by_name("output_node:0") output = sess.run(output_node) self.assertNear(2.0, output, 0.00001)
def main(_): # Set up the pre-trained graph. maybe_download_and_extract() graph, bottleneck_tensor, jpeg_data_tensor, resized_image_tensor = ( create_inception_graph()) # Look at the folder structure, and create lists of all the images. image_lists = create_image_lists(FLAGS.image_dir, FLAGS.testing_percentage, FLAGS.validation_percentage) class_count = len(image_lists.keys()) if class_count == 0: print('No valid folders of images found at ' + FLAGS.image_dir) return -1 if class_count == 1: print('Only one valid folder of images found at ' + FLAGS.image_dir + ' - multiple classes are needed for classification.') return -1 # See if the command-line flags mean we're applying any distortions. do_distort_images = should_distort_images(FLAGS.flip_left_right, FLAGS.random_crop, FLAGS.random_scale, FLAGS.random_brightness) sess = tf.Session() if do_distort_images: # We will be applying distortions, so setup the operations we'll need. distorted_jpeg_data_tensor, distorted_image_tensor = add_input_distortions( FLAGS.flip_left_right, FLAGS.random_crop, FLAGS.random_scale, FLAGS.random_brightness) else: # We'll make sure we've calculated the 'bottleneck' image summaries and # cached them on disk. cache_bottlenecks(sess, image_lists, FLAGS.image_dir, FLAGS.bottleneck_dir, jpeg_data_tensor, bottleneck_tensor) # Add the new layer that we'll be training. (train_step, cross_entropy, bottleneck_input, ground_truth_input, final_tensor) = add_final_training_ops(len(image_lists.keys()), FLAGS.final_tensor_name, bottleneck_tensor) # Set up all our weights to their initial default values. init = tf.initialize_all_variables() sess.run(init) # Create the operations we need to evaluate the accuracy of our new layer. evaluation_step = add_evaluation_step(final_tensor, ground_truth_input) # Run the training for as many cycles as requested on the command line. for i in range(FLAGS.how_many_training_steps): # Get a catch of input bottleneck values, either calculated fresh every time # with distortions applied, or from the cache stored on disk. if do_distort_images: train_bottlenecks, train_ground_truth = get_random_distorted_bottlenecks( sess, image_lists, FLAGS.train_batch_size, 'training', FLAGS.image_dir, distorted_jpeg_data_tensor, distorted_image_tensor, resized_image_tensor, bottleneck_tensor) else: train_bottlenecks, train_ground_truth = get_random_cached_bottlenecks( sess, image_lists, FLAGS.train_batch_size, 'training', FLAGS.bottleneck_dir, FLAGS.image_dir, jpeg_data_tensor, bottleneck_tensor) # Feed the bottlenecks and ground truth into the graph, and run a training # step. sess.run(train_step, feed_dict={ bottleneck_input: train_bottlenecks, ground_truth_input: train_ground_truth }) # Every so often, print out how well the graph is training. is_last_step = (i + 1 == FLAGS.how_many_training_steps) if (i % FLAGS.eval_step_interval) == 0 or is_last_step: train_accuracy, cross_entropy_value = sess.run( [evaluation_step, cross_entropy], feed_dict={ bottleneck_input: train_bottlenecks, ground_truth_input: train_ground_truth }) print('%s: Step %d: Train accuracy = %.1f%%' % (datetime.now(), i, train_accuracy * 100)) print('%s: Step %d: Cross entropy = %f' % (datetime.now(), i, cross_entropy_value)) validation_bottlenecks, validation_ground_truth = ( get_random_cached_bottlenecks( sess, image_lists, FLAGS.validation_batch_size, 'validation', FLAGS.bottleneck_dir, FLAGS.image_dir, jpeg_data_tensor, bottleneck_tensor)) validation_accuracy = sess.run(evaluation_step, feed_dict={ bottleneck_input: validation_bottlenecks, ground_truth_input: validation_ground_truth }) print('%s: Step %d: Validation accuracy = %.1f%%' % (datetime.now(), i, validation_accuracy * 100)) # We've completed all our training, so run a final test evaluation on # some new images we haven't used before. test_bottlenecks, test_ground_truth = get_random_cached_bottlenecks( sess, image_lists, FLAGS.test_batch_size, 'testing', FLAGS.bottleneck_dir, FLAGS.image_dir, jpeg_data_tensor, bottleneck_tensor) test_accuracy = sess.run(evaluation_step, feed_dict={ bottleneck_input: test_bottlenecks, ground_truth_input: test_ground_truth }) print('Final test accuracy = %.1f%%' % (test_accuracy * 100)) # Write out the trained graph and labels with the weights stored as constants. output_graph_def = graph_util.convert_variables_to_constants( sess, graph.as_graph_def(), [FLAGS.final_tensor_name]) with gfile.FastGFile(FLAGS.output_graph, 'wb') as f: f.write(output_graph_def.SerializeToString()) with gfile.FastGFile(FLAGS.output_labels, 'w') as f: f.write('\n'.join(image_lists.keys()) + '\n')
def main(_): # Set up the pre-trained graph. InputManager.maybe_download_and_extract(model_dir=FLAGS.model_dir, data_url=DATA_URL) (graph, bottleneck_tensor, jpeg_data_tensor, resized_image_tensor) = GraphBuilder.create_inception_graph( model_dir=FLAGS.model_dir, bottleneck_tensor_name=BOTTLENECK_TENSOR_NAME, jpeg_data_tensor_name=JPEG_DATA_TENSOR_NAME, resized_input_tensor_name=RESIZED_INPUT_TENSOR_NAME) # Look at the folder structure, and create lists of all the images. image_lists = InputManager.create_image_lists( image_dir=FLAGS.image_dir, testing_percentage=FLAGS.testing_percentage, validation_percentage=FLAGS.validation_percentage) class_count = len(image_lists.keys()) if class_count == 0: print('No valid folders of images found at ' + FLAGS.image_dir) return -1 if class_count == 1: print('Only one valid folder of images found at ' + FLAGS.image_dir + ' - multiple classes are needed for classification.') return -1 # See if the command-line flags mean we're applying any distortions. do_distort_images = utils.should_distort_images( flip_left_right=FLAGS.flip_left_right, random_crop=FLAGS.random_crop, random_scale=FLAGS.random_scale, random_brightness=FLAGS.random_brightness) sess = tf.Session() if do_distort_images: # We will be applying distortions, so setup the operations we'll need. (distorted_jpeg_data_tensor, distorted_image_tensor) = GraphBuilder.add_input_distortions( model_input_depth=MODEL_INPUT_DEPTH, model_input_width=MODEL_INPUT_WIDTH, model_input_height=MODEL_INPUT_HEIGHT, flip_left_right=FLAGS.flip_left_right, random_crop=FLAGS.random_crop, random_scale=FLAGS.random_scale, random_brightness=FLAGS.random_brightness) else: # We'll make sure we've calculated the 'bottleneck' image summaries and cached them on disk. Bottleneck.cache_bottlenecks(sess=sess, image_lists=image_lists, image_dir=FLAGS.image_dir, bottleneck_dir=FLAGS.bottleneck_dir, jpeg_data_tensor=jpeg_data_tensor, bottleneck_tensor=bottleneck_tensor) # Add the new layer that we'll be training. (train_step, cross_entropy, bottleneck_input, ground_truth_input, final_tensor) = GraphBuilder.add_final_training_ops( bottleneck_tensor_size=BOTTLENECK_TENSOR_SIZE, learning_rate=FLAGS.learning_rate, class_count=len(image_lists.keys()), final_tensor_name=FLAGS.final_tensor_name, bottleneck_tensor=bottleneck_tensor) # Merge all the summaries and write them out to /tmp/transfer_learning_inception merged = tf.merge_all_summaries() train_writer = tf.train.SummaryWriter(FLAGS.summaries_dir + '/train', sess.graph) validation_writer = tf.train.SummaryWriter(FLAGS.summaries_dir + '/validation') # Set up all our weights to their initial default values. init = tf.initialize_all_variables() sess.run(init) # Create the operations we need to evaluate the accuracy of our new layer. evaluation_step = GraphBuilder.add_evaluation_step(final_tensor, ground_truth_input) # Run the training for as many cycles as requested on the command line. for i in range(FLAGS.how_many_training_steps): # Get a catch of input bottleneck values, either calculated fresh every time with distortions applied, # or from the cache stored on disk. if do_distort_images: train_bottlenecks, train_ground_truth = Bottleneck.get_random_distorted_bottlenecks( sess, image_lists, FLAGS.train_batch_size, 'training', FLAGS.image_dir, distorted_jpeg_data_tensor, distorted_image_tensor, resized_image_tensor, bottleneck_tensor) else: train_bottlenecks, train_ground_truth = Bottleneck.get_random_cached_bottlenecks( sess, image_lists, FLAGS.train_batch_size, 'training', FLAGS.bottleneck_dir, FLAGS.image_dir, jpeg_data_tensor, bottleneck_tensor) # Feed the bottlenecks and ground truth into the graph, and run a training step. summary, _ = sess.run( [merged, train_step], feed_dict={ bottleneck_input: train_bottlenecks, ground_truth_input: train_ground_truth }) # Add summaries to train writer train_writer.add_summary(summary, i) # Every so often, print out how well the graph is training. is_last_step = (i + 1 == FLAGS.how_many_training_steps) if (i % FLAGS.eval_step_interval) == 0 or is_last_step: train_accuracy, cross_entropy_value = sess.run( [evaluation_step, cross_entropy], feed_dict={ bottleneck_input: train_bottlenecks, ground_truth_input: train_ground_truth }) print('%s: Step %d: Train accuracy = %.1f%%' % (datetime.now(), i, train_accuracy * 100)) print('%s: Step %d: Cross entropy = %f' % (datetime.now(), i, cross_entropy_value)) validation_bottlenecks, validation_ground_truth = ( Bottleneck.get_random_cached_bottlenecks( sess, image_lists, FLAGS.validation_batch_size, 'validation', FLAGS.bottleneck_dir, FLAGS.image_dir, jpeg_data_tensor, bottleneck_tensor)) summary, validation_accuracy = sess.run( [merged, evaluation_step], feed_dict={ bottleneck_input: validation_bottlenecks, ground_truth_input: validation_ground_truth }) # Add summaries to validation writer validation_writer.add_summary(summary, i) print('%s: Step %d: Validation accuracy = %.1f%%' % (datetime.now(), i, validation_accuracy * 100)) # We've completed all our training, so run a final test evaluation on some new images we haven't used before. test_bottlenecks, test_ground_truth = Bottleneck.get_random_cached_bottlenecks( sess, image_lists, FLAGS.test_batch_size, 'testing', FLAGS.bottleneck_dir, FLAGS.image_dir, jpeg_data_tensor, bottleneck_tensor) test_accuracy = sess.run(evaluation_step, feed_dict={ bottleneck_input: test_bottlenecks, ground_truth_input: test_ground_truth }) print('Final test accuracy = %.1f%%' % (test_accuracy * 100)) # Write out the trained graph and labels with the weights stored as constants. output_graph_def = graph_util.convert_variables_to_constants( sess, graph.as_graph_def(), [FLAGS.final_tensor_name]) with gfile.FastGFile(FLAGS.output_graph, 'wb') as f: f.write(output_graph_def.SerializeToString()) with gfile.FastGFile(FLAGS.output_labels, 'w') as f: f.write('\n'.join(image_lists.keys()) + '\n')
def run_training(): """Train fully_connected for a number of steps.""" # Get the sets of images and labels for training, validation, and # test on fully_connected. # Look at the folder structure, and create lists of all the images. data_sets = regression_data.create_image_lists(FLAGS.image_dir, FLAGS.testing_percentage, FLAGS.validation_percentage) # Tell TensorFlow that the model will be built into the default Graph. with tf.Graph().as_default(): # Generate placeholders for the images and labels. images_placeholder, labels_placeholder = placeholder_inputs( FLAGS.batch_size, name="input_images") # Build a Graph that computes predictions from the inference model. inference = regression.inference(images_placeholder, FLAGS.hidden1, FLAGS.hidden2) # logits = fully_connected.inference(images_placeholder, # FLAGS.hidden1, # FLAGS.hidden2) # softmax = tf.nn.softmax(logits, name="final_result") # Add to the Graph the Ops for loss calculation. loss = regression.loss(inference, labels_placeholder) # Add to the Graph the Ops that calculate and apply gradients. train_op = regression.training(loss, FLAGS.learning_rate) # Add the Op to compare the logits to the labels during evaluation. eval_correct = regression.evaluation(inference, labels_placeholder) # Build the summary operation based on the TF collection of Summaries. summary_op = tf.merge_all_summaries() # Create a saver for writing training checkpoints. saver = tf.train.Saver() # Create a session for running Ops on the Graph. sess = tf.Session() # graph = sess.graph # Run the Op to initialize the variables. init = tf.initialize_all_variables() sess.run(init) # Instantiate a SummaryWriter to output summaries and the Graph. summary_writer = tf.train.SummaryWriter(FLAGS.train_dir, graph_def=sess.graph_def) # And then after everything is built, start the training loop. for step in xrange(FLAGS.max_steps): start_time = time.time() # Fill a feed dictionary with the actual set of images and labels # for this particular training step. feed_dict = fill_feed_dict(data_sets.train, images_placeholder, labels_placeholder) # Run one step of the model. The return values are the activations # from the `train_op` (which is discarded) and the `loss` Op. To # inspect the values of your Ops or variables, you may include them # in the list passed to sess.run() and the value tensors will be # returned in the tuple from the call. _, loss_value, inference_val = sess.run([train_op, loss, inference], feed_dict=feed_dict) duration = time.time() - start_time # Write the summaries and print an overview fairly often. if step % 10 == 0: # Print status to stdout. print('Step %d: loss = %.6f (%.3f sec)' % (step, loss_value, duration)) # Update the events file. summary_str = sess.run(summary_op, feed_dict=feed_dict) summary_writer.add_summary(summary_str, step) # Save a checkpoint and evaluate the model periodically. if (step + 1) % 1000 == 0 or (step + 1) == FLAGS.max_steps: # saver.save(sess, FLAGS.train_dir, global_step=step) # Evaluate against the training set. # print('Training Data Eval:') # do_eval(sess, # eval_correct, # images_placeholder, # labels_placeholder, # data_sets.train) # Evaluate against the validation set. # print('Validation Data Eval:') # do_eval(sess, # eval_correct, # images_placeholder, # labels_placeholder, # data_sets.validation) # Evaluate against the test set. # print('Test Data Eval:') # do_eval(sess, # eval_correct, # images_placeholder, # labels_placeholder, # data_sets.test) # Write out the trained graph and labels with the weights stored as constants. output_graph_def = graph_util.convert_variables_to_constants( sess, sess.graph.as_graph_def(), ["final_result"]) with gfile.FastGFile("output_graph.pb", 'wb') as f: f.write(output_graph_def.SerializeToString())
def train_and_freeze_graph(): ##################################################################################### # train a LMS model # generate 100 samples with Y = W * X + b, where X, Y are 2-D column vectors, # and W and b have following values: # / 1.0 0.0 \ / 1.0 \ # W = | | , b = | | # \ 0.0, 2.0 / \ 0.5 / W_real = np.array([[1.0, 0.0], [0.0, 2.0]]) b_real = np.array([[1.0], [0.5]]) X_data = np.random.rand(2, 100).astype(np.float32) Y_data = np.dot(W_real, X_data) + b_real # Try to find values for W and b that compute y_data = W * x_data + b W = tf.Variable(tf.random_uniform([2, 2], -1.0, 1.0), name = "W_learn") b = tf.Variable(tf.zeros([2, 1]), name = "b_learn") x = tf.placeholder(tf.float32, [2, None], name = "x_input") y = tf.add(tf.matmul(W, x), b, name = "y_guess") # Minimize the mean squared errors. loss = tf.reduce_mean(tf.square(y - Y_data)) optimizer = tf.train.GradientDescentOptimizer(0.5) grads = optimizer.compute_gradients(loss) apply_grad_op = optimizer.apply_gradients(grads) # Before starting, initialize the variables. We will 'run' this first. init = tf.initialize_all_variables() # Launch the graph. sess = tf.Session() sess.run(init) for i in xrange(200): _, loss_value = sess.run([apply_grad_op, loss], feed_dict={x: X_data}) print("loss %f" % loss_value) ## gradients has the same shape as it's corresponding variable if i % 50 == 0: print("=======================================") print("type of grads: %s, size %d" % (type(grads), len(grads))) print("type of grad : %s, type of var %s" % (type(grads[0][0]), type(grads[0][1]))) for grad, var in grads: print("gradients of %s:" % var.name) print(sess.run(grad, feed_dict={x: X_data})) print("=======================================") W_value, b_value = sess.run([W, b]) print("after training:") print("W is:") print(W_value) print("b is:") print(b_value) print("x.name %s" % x.name) print("y.name %s" % y.name) ##################################################################################### ##################################################################################### # freeze and save the trained model for later restoring # Write out the trained graph and labels with the weights stored as constants. output_graph_def = graph_util.convert_variables_to_constants( sess, sess.graph.as_graph_def(), ["x_input", "y_guess"]) with gfile.FastGFile('draft_graph.pb', 'wb') as f: f.write(output_graph_def.SerializeToString())
def learn(data_folder, experts, learning_rate=.001, train_ratio=.8, validation_ratio=.1, test_ratio=.1, save_every=10, batch_size=2048, hidden_size=1024, dropout=.5, epochs=500, print_every=1, model_dir='.', perceptron=False, mem_ratio=.95): assert train_ratio + validation_ratio + test_ratio == 1, 'Train/validation/test ratios must sum up to 1' data = read_data(data_folder, train_ratio, validation_ratio, test_ratio) model_name = ( '''transfer_classifier_moe_epochs_{}_batch_{}_ratios_{}_{}_{}_''' '''learning_rate_{}'''.format(epochs, batch_size, train_ratio, validation_ratio, test_ratio, learning_rate)) if perceptron: model_name = '{}_perceptron.pb'.format(model_name) else: model_name = '{}_dropout_{}_hidden_size_{}.pb'.format( model_name, dropout, hidden_size) gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=mem_ratio) with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess: local_experts = {} for model in os.listdir(experts): print('Loading {}'.format(model)) load_graph(os.path.join(args.experts, model)) stripped = model[20:] h5 = stripped[:stripped.find('_')] # MESSY. local_experts[h5] = sess.graph.get_tensor_by_name( '{}output:0'.format(h5)) data.train._X = np.hstack([ data.train.X, np.vstack([ flow(sess, local_experts, x) for x in chunks(data.train.X, batch_size) ]) ]) data.validation._X = np.hstack( [data.validation.X, flow(sess, local_experts, data.validation.X)]) data.test._X = np.hstack( [data.test.X, flow(sess, local_experts, data.test.X)]) x = tf.placeholder('float', shape=[None, data.train.X_features], name='input') y_ = tf.placeholder('float', shape=[None, data.train.Y_features], name='target') if perceptron: W = weight_variable([data.train.X_features, data.train.Y_features], name='weights') b = bias_variable([data.train.Y_features], name='bias') logits = tf.matmul(x, W) + b else: W_in = weight_variable([data.train.X_features, hidden_size], name='weights_in') b_in = bias_variable([hidden_size], name='bias_in') hidden = tf.matmul(x, W_in) + b_in relu = tf.nn.relu(hidden) keep_prob = tf.placeholder_with_default([1.], shape=None) hidden_dropout = tf.nn.dropout(relu, keep_prob) W_out = weight_variable([hidden_size, data.train.Y_features], name='weights_out') b_out = bias_variable([data.train.Y_features], name='bias_out') logits = tf.matmul(relu, W_out) + b_out y = tf.nn.softmax(logits, name='output') cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits, y_) train_step = tf.train.AdamOptimizer(learning_rate).minimize( cross_entropy) correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, 'float')) sess.run(tf.initialize_all_variables()) last_epoch = 0 t_epoch = time.time() while data.train.epoch <= epochs: epoch = data.train.epoch batch_x, batch_y = data.train.next_batch(batch_size) t_start = time.time() feed_dict = { x: batch_x, y_: batch_y } if perceptron else { x: batch_x, y_: batch_y, keep_prob: dropout } train_step.run(feed_dict=feed_dict) t_end = time.time() - t_start if epoch > last_epoch: if epoch % print_every == 0: train_accuracy = accuracy.eval(feed_dict={ x: batch_x, y_: batch_y }) validation_accuracy = accuracy.eval(feed_dict={ x: data.validation.X, y_: data.validation.Y }) print( '''Epoch {} train accuracy: {}, validation accuracy: {}. ''' '''{} states/sec, {} secs/epoch.'''.format( epoch, train_accuracy, validation_accuracy, batch_size / t_end, time.time() - t_epoch)) if epoch % save_every == 0 or epoch == epochs: output_graph_def = graph_util.convert_variables_to_constants( sess, sess.graph.as_graph_def(), ['input', 'output']) with gfile.FastGFile(os.path.join(model_dir, model_name), 'w') as f: f.write(output_graph_def.SerializeToString()) t_epoch = time.time() last_epoch = epoch print('Trained model saved to {}'.format( os.path.join(model_dir, model_name))) if test_ratio > 0: test_accuracy = accuracy.eval(feed_dict={ x: data.test.X, y_: data.test.Y }) print('Evaluation on testing data: {}'.format(test_accuracy))
def learn( train_states, test_states, model, learning_rate=0.0001, save_every=10, batch_size=2048, hidden_size=2048, dropout=0.5, epochs=500, print_every=1, model_dir=".", perceptron=False, mem_ratio=0.95, ): data = read_data(train_states, test_states) model_name = """trust_classifier_epochs_{}_batch_{}_learning_rate_{}""".format(epochs, batch_size, learning_rate) if perceptron: model_name = "{}_perceptron.pb".format(model_name) else: model_name = "{}_dropout_{}_hidden_size_{}.pb".format(model_name, dropout, hidden_size) gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=mem_ratio) with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess: load_graph(model) transfer_predictor = sess.graph.get_tensor_by_name("output:0") # Evaluate the model on the training and test data, for training and testing. data.train._X = np.vstack( [sess.run(transfer_predictor, {"input:0": chunk}) for chunk in chunks(data.train.X, 10)] ) answer = tf.equal(tf.argmax(data.train.X, 1), tf.argmax(data.train.Y, 1)) data.train._Y = tf.one_hot(tf.to_int32(answer), depth=2).eval() data.test._X = sess.run(transfer_predictor, {"input:0": data.test.X}) answer = tf.equal(tf.argmax(data.test.X, 1), tf.argmax(data.test.Y, 1)) data.test._Y = tf.one_hot(tf.to_int32(answer), depth=2).eval() x = tf.placeholder("float", shape=[None, data.train.X_features], name="input_b") y_ = tf.placeholder("float", shape=[None, data.train.Y_features], name="target") if perceptron: W = weight_variable([data.train.X_features, data.train.Y_features], name="weights") b = bias_variable([data.train.Y_features], name="bias") logits = tf.matmul(x, W) + b else: W_in = weight_variable([data.train.X_features, hidden_size], name="weights_in") b_in = bias_variable([hidden_size], name="bias_in") hidden = tf.matmul(x, W_in) + b_in relu = tf.nn.relu(hidden) keep_prob = tf.placeholder_with_default([1.0], shape=None) hidden_dropout = tf.nn.dropout(relu, keep_prob) W_out = weight_variable([hidden_size, data.train.Y_features], name="weights_out") b_out = bias_variable([data.train.Y_features], name="bias_out") logits = tf.matmul(relu, W_out) + b_out # Loss & train cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits, y_) train_step = tf.train.AdamOptimizer(learning_rate).minimize(cross_entropy) # Evaluation y = tf.nn.softmax(logits, name="output_b") correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float")) sess.run(tf.initialize_all_variables()) last_epoch = 0 t_epoch = time.time() while data.train.epoch <= epochs: epoch = data.train.epoch batch_x, batch_y = data.train.next_batch(batch_size) t_start = time.time() feed_dict = {x: batch_x, y_: batch_y} if perceptron else {x: batch_x, y_: batch_y, keep_prob: dropout} train_step.run(feed_dict=feed_dict) t_end = time.time() - t_start if epoch > last_epoch: if epoch % print_every == 0: train_accuracy_mean = accuracy.eval(feed_dict={x: batch_x, y_: batch_y}) validation_accuracy_mean = accuracy.eval(feed_dict={x: data.test.X, y_: data.test.Y}) print( """Epoch {} train accuracy: {}, test accuracy: {}. """ """{} states/sec, {} secs/epoch.""".format( epoch, train_accuracy_mean, validation_accuracy_mean, batch_size / t_end, time.time() - t_epoch, ) ) if epoch % save_every == 0 or epoch == epochs: output_graph_def = graph_util.convert_variables_to_constants( sess, sess.graph.as_graph_def(), ["input_b", "output_b"] ) with gfile.FastGFile(os.path.join(model_dir, model_name), "w") as f: f.write(output_graph_def.SerializeToString()) t_epoch = time.time() last_epoch = epoch print("Trained model saved to {}".format(os.path.join(model_dir, model_name)))
def learn(train_states, test_states, model, learning_rate=.0001, save_every=10, batch_size=2048, hidden_size=2048, dropout=.5, epochs=500, print_every=1, model_dir='.', perceptron=False, mem_ratio=.95): data = read_data(train_states, test_states) model_name = ( '''trust_classifier_epochs_{}_batch_{}_learning_rate_{}'''.format( epochs, batch_size, learning_rate)) if perceptron: model_name = '{}_perceptron.pb'.format(model_name) else: model_name = '{}_dropout_{}_hidden_size_{}.pb'.format( model_name, dropout, hidden_size) gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=mem_ratio) with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess: load_graph(model) transfer_predictor = sess.graph.get_tensor_by_name('output:0') # Evaluate the model on the training and test data, for training and testing. data.train._X = np.vstack([ sess.run(transfer_predictor, {'input:0': chunk}) for chunk in chunks(data.train.X, 10) ]) answer = tf.equal(tf.argmax(data.train.X, 1), tf.argmax(data.train.Y, 1)) data.train._Y = tf.one_hot(tf.to_int32(answer), depth=2).eval() data.test._X = sess.run(transfer_predictor, {'input:0': data.test.X}) answer = tf.equal(tf.argmax(data.test.X, 1), tf.argmax(data.test.Y, 1)) data.test._Y = tf.one_hot(tf.to_int32(answer), depth=2).eval() x = tf.placeholder('float', shape=[None, data.train.X_features], name='input_b') y_ = tf.placeholder('float', shape=[None, data.train.Y_features], name='target') if perceptron: W = weight_variable([data.train.X_features, data.train.Y_features], name='weights') b = bias_variable([data.train.Y_features], name='bias') logits = tf.matmul(x, W) + b else: W_in = weight_variable([data.train.X_features, hidden_size], name='weights_in') b_in = bias_variable([hidden_size], name='bias_in') hidden = tf.matmul(x, W_in) + b_in relu = tf.nn.relu(hidden) keep_prob = tf.placeholder_with_default([1.], shape=None) hidden_dropout = tf.nn.dropout(relu, keep_prob) W_out = weight_variable([hidden_size, data.train.Y_features], name='weights_out') b_out = bias_variable([data.train.Y_features], name='bias_out') logits = tf.matmul(relu, W_out) + b_out # Loss & train cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits, y_) train_step = tf.train.AdamOptimizer(learning_rate).minimize( cross_entropy) # Evaluation y = tf.nn.softmax(logits, name='output_b') correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, 'float')) sess.run(tf.initialize_all_variables()) last_epoch = 0 t_epoch = time.time() while data.train.epoch <= epochs: epoch = data.train.epoch batch_x, batch_y = data.train.next_batch(batch_size) t_start = time.time() feed_dict = { x: batch_x, y_: batch_y } if perceptron else { x: batch_x, y_: batch_y, keep_prob: dropout } train_step.run(feed_dict=feed_dict) t_end = time.time() - t_start if epoch > last_epoch: if epoch % print_every == 0: train_accuracy_mean = accuracy.eval(feed_dict={ x: batch_x, y_: batch_y }) validation_accuracy_mean = accuracy.eval(feed_dict={ x: data.test.X, y_: data.test.Y }) print( '''Epoch {} train accuracy: {}, test accuracy: {}. ''' '''{} states/sec, {} secs/epoch.'''.format( epoch, train_accuracy_mean, validation_accuracy_mean, batch_size / t_end, time.time() - t_epoch)) if epoch % save_every == 0 or epoch == epochs: output_graph_def = graph_util.convert_variables_to_constants( sess, sess.graph.as_graph_def(), ['input_b', 'output_b']) with gfile.FastGFile(os.path.join(model_dir, model_name), 'w') as f: f.write(output_graph_def.SerializeToString()) t_epoch = time.time() last_epoch = epoch print('Trained model saved to {}'.format( os.path.join(model_dir, model_name)))
def learn(train_states, test_states, learning_rate, save_every, batch_size, hidden_size, dropout, epochs, print_every, model_dir, perceptron, dense, lenet, filter_width, depth, mem_ratio, q_size, use_dask, in_memory, dask_chunksize): data = read_data(train_states, test_states, use_dask, in_memory, dask_chunksize) model_name = ('''transfer_classifier_epochs_{}_batch_{}_learning_rate_{}'''.format( epochs, batch_size, learning_rate)) if perceptron: model_name = '{}_perceptron.pb'.format(model_name) if dense: model_name = '{}_dense_dropout_{}_hidden_size_{}.pb'.format(model_name, dropout, hidden_size) if lenet: model_name = '{}_lenet_dropout_{}_hidden_size_{}.pb'.format(model_name, dropout, hidden_size) gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=mem_ratio) with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess: q_x_in = tf.placeholder(tf.float32, shape=[batch_size, data.train.X_features]) q_y_in = tf.placeholder(tf.int32, shape=[batch_size]) y_real = tf.placeholder(tf.int32, shape=[None]) keep_prob = tf.placeholder_with_default([1.], shape=None) q = tf.FIFOQueue(q_size, [tf.float32, tf.int32], shapes=[ q_x_in.get_shape(), q_y_in.get_shape()]) enqueue_op = q.enqueue([q_x_in, q_y_in]) q_x_out, q_y_out = q.dequeue() x = tf.placeholder_with_default(q_x_out, shape=[None, data.train.X_features], name='input') y_ = tf.placeholder_with_default(q_y_out, shape=[None]) if perceptron: W = weight_variable([data.train.X_features, data.train.Y_features]) b = bias_variable([data.train.Y_features]) logits = tf.matmul(x, W) + b if dense: W_in = weight_variable([data.train.X_features, hidden_size]) b_in = bias_variable([hidden_size]) hidden = tf.matmul(x, W_in) + b_in relu = tf.nn.relu(hidden) hidden_dropout = tf.nn.dropout(relu, keep_prob) W_out = weight_variable([hidden_size,data.train.Y_features]) b_out = bias_variable([data.train.Y_features]) logits = tf.matmul(hidden_dropout, W_out) + b_out if lenet: w1 = weight_variable([1, filter_width, 1, depth]) b1 = bias_variable([depth]) x_4d = tf.expand_dims(tf.expand_dims(x,1),-1) # Singleton dimension height, out_channel conv = tf.nn.conv2d(x_4d, w1, strides=[1,1,1,1], padding='SAME') relu = tf.nn.relu(tf.nn.bias_add(conv, b1)) pool = tf.nn.max_pool(relu, ksize=[1, 1, 2, 1], strides=[1, 1, 2, 1], padding='SAME') w2 = weight_variable([1, filter_width, depth, depth*2]) b2 = bias_variable([depth*2]) conv = tf.nn.conv2d(pool, w2, strides=[1,1,1,1], padding='SAME') relu = tf.nn.relu(tf.nn.bias_add(conv, b2)) pool = tf.nn.max_pool(relu, ksize=[1, 1, 2, 1], strides=[1, 1, 2, 1], padding='SAME') pool_shape = tf.shape(pool) reshape = tf.reshape(pool, [pool_shape[0], pool_shape[1] * pool_shape[2] * pool_shape[3] ]) w3 = weight_variable([ (data.train.X_features/4)*2*depth, hidden_size]) b3 = bias_variable([hidden_size]) hidden = tf.matmul(reshape, w3) + b3 relu = tf.nn.relu(hidden) hidden_dropout = tf.nn.dropout(relu, keep_prob) w4 = weight_variable([hidden_size, data.train.Y_features]) b4 = bias_variable([data.train.Y_features]) logits = tf.matmul(hidden_dropout, w4) + b4 # Loss & train cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits, y_) train_step = tf.train.AdamOptimizer(learning_rate).minimize(cross_entropy) # Evaluation y = tf.nn.softmax(logits) train_correct_prediction = tf.equal(tf.to_int32(tf.argmax(y, 1)), y_,) train_accuracy = tf.reduce_mean(tf.cast(train_correct_prediction, tf.float32)) # Applying convolutional filter to put subcategories into original categories for testing stride = [1,1,1,1] _filter = tf.constant(data.output_filter, dtype=tf.float32, shape=data.output_filter.shape) conv_in = tf.expand_dims(y, 0) conv_in = tf.expand_dims(conv_in,-1) conv_out = tf.nn.conv2d(conv_in, _filter, stride, 'VALID') # We don't want zero padding. back = tf.squeeze(conv_out, squeeze_dims=[0,2], name='output') test_correct_prediction = tf.equal(tf.to_int32(tf.argmax(back, 1)), y_real) test_accuracy = tf.reduce_mean(tf.cast(test_correct_prediction, tf.float32)) sess.run(tf.initialize_all_variables()) def load_data(): try: while True: next_x, next_y = data.train.next_batch(batch_size) if next_x.shape[0] == batch_size: sess.run(enqueue_op, feed_dict={q_x_in: next_x, q_y_in: next_y}) except Exception as error: print(error) print('Stopped streaming of data.') data_thread = threading.Thread(target=load_data) data_thread.daemon = True data_thread.start() last_epoch = 0 epoch = 0 t_epoch = time.time() t_end = [] i = 0 while epoch <= epochs: t_start = time.time() sess.run(train_step, feed_dict={keep_prob: dropout}) t_end.append(time.time() - t_start) if epoch > last_epoch: if epoch % print_every == 0: batch_x, batch_y = data.train.next_batch(batch_size) train_accuracy_mean = train_accuracy.eval(feed_dict={ x: batch_x, y_: batch_y }) validation_accuracy_mean = np.mean([ test_accuracy.eval(feed_dict={x: t_x, y_real: t_y }) for t_x, t_y in zip(chunks(data.test.X, batch_size), chunks(data.test.Y, batch_size)) ]) print('''Epoch {} train accuracy: {}, test accuracy: {}. ''' '''{} states/sec on average, {} secs/epoch.'''.format(epoch, pf(train_accuracy_mean), pf(validation_accuracy_mean), pf(batch_size/np.mean(t_end)), pf(time.time() - t_epoch))) if epoch % save_every == 0 or epoch == epochs: output_graph_def = graph_util.convert_variables_to_constants( sess, sess.graph.as_graph_def(), ['input', 'output']) with gfile.FastGFile(os.path.join(model_dir, model_name), 'w') as f: f.write(output_graph_def.SerializeToString()) t_epoch = time.time() t_end = [] last_epoch = epoch i += 1 if batch_size*i > data.train.X_len: epoch += 1 i = 0 q.close(cancel_pending_enqueues=True) print('Trained model saved to {}'.format(os.path.join(model_dir, model_name)))
from tensorflow.python.client import graph_util v1 = tf.Variable(tf.constant(1.0, shape=[1]), name = "v1") v2 = tf.Variable(tf.constant(2.0, shape=[1]), name = "v2") result = v1 + v2 print result v3 = tf.Variable(tf.constant(3.0, shape=[1]), name = "v1") v4 = tf.Variable(tf.constant(4.0, shape=[1]), name = "v2") result = v3 + v4 print result init_op = tf.initialize_all_variables() with tf.Session() as sess: sess.run(init_op) graph_def = tf.get_default_graph().as_graph_def() output_graph_def = graph_util.convert_variables_to_constants(sess, graph_def, ['add_1']) with tf.gfile.GFile("../../datasets/combined_model.pb", "wb") as f: f.write(output_graph_def.SerializeToString()) from tensorflow.python.platform import gfile with tf.Session() as sess: model_filename = "../../datasets/combined_model.pb" with gfile.FastGFile(model_filename, 'rb1') as f: graph_def = tf.GraphDef() graph_def.ParseFromString(f.read()) result = tf.import_graph_def(graph_def, return_elements=["add_1:0"]) print sess.run(result)
def main(_): # Set up the pre-trained graph. maybe_download_and_extract() graph, bottleneck_tensor, jpeg_data_tensor, resized_image_tensor = ( create_inception_graph()) # Get drviers list imagesTodrivers=get_driver_data() # Look at the folder structure, and create lists of all the images. image_lists = create_image_lists(FLAGS.image_dir, FLAGS.testing_percentage, FLAGS.validation_percentage, imagesTodrivers, driversToPutInTesting, driversToPutInCV, usingProcessed) class_count = len(image_lists.keys()) if class_count == 0: print('No valid folders of images found at ' + FLAGS.image_dir) return -1 if class_count == 1: print('Only one valid folder of images found at ' + FLAGS.image_dir + ' - multiple classes are needed for classification.') return -1 # See if the command-line flags mean we're applying any distortions. do_distort_images = should_distort_images( FLAGS.flip_left_right, FLAGS.random_crop, FLAGS.random_scale, FLAGS.random_brightness) sess = tf.Session() if do_distort_images: # We will be applying distortions, so setup the operations we'll need. distorted_jpeg_data_tensor, distorted_image_tensor = add_input_distortions( FLAGS.flip_left_right, FLAGS.random_crop, FLAGS.random_scale, FLAGS.random_brightness) else: # We'll make sure we've calculated the 'bottleneck' image summaries and # cached them on disk. cache_bottlenecks(sess, image_lists, FLAGS.image_dir, FLAGS.bottleneck_dir, jpeg_data_tensor, bottleneck_tensor) # Add the new layer that we'll be training. (train_step, cross_entropy, bottleneck_input, ground_truth_input, final_tensor) = add_final_training_ops(len(image_lists.keys()), FLAGS.final_tensor_name, bottleneck_tensor) # Set up all our weights to their initial default values. init = tf.initialize_all_variables() sess.run(init) # Create the operations we need to evaluate the accuracy of our new layer. evaluation_step = add_evaluation_step(final_tensor, ground_truth_input) # Run the training for as many cycles as requested on the command line. for i in range(FLAGS.how_many_training_steps): # Get a catch of input bottleneck values, either calculated fresh every time # with distortions applied, or from the cache stored on disk. if do_distort_images: train_bottlenecks, train_ground_truth = get_random_distorted_bottlenecks( sess, image_lists, FLAGS.train_batch_size, 'training', FLAGS.image_dir, distorted_jpeg_data_tensor, distorted_image_tensor, resized_image_tensor, bottleneck_tensor) else: train_bottlenecks, train_ground_truth = get_random_cached_bottlenecks( sess, image_lists, FLAGS.train_batch_size, 'training', FLAGS.bottleneck_dir, FLAGS.image_dir, jpeg_data_tensor, bottleneck_tensor) # Feed the bottlenecks and ground truth into the graph, and run a training # step. sess.run(train_step, feed_dict={bottleneck_input: train_bottlenecks, ground_truth_input: train_ground_truth}) # Every so often, print out how well the graph is training. is_last_step = (i + 1 == FLAGS.how_many_training_steps) if (i % FLAGS.eval_step_interval) == 0 or is_last_step: train_accuracy, cross_entropy_value = sess.run( [evaluation_step, cross_entropy], feed_dict={bottleneck_input: train_bottlenecks, ground_truth_input: train_ground_truth}) print('%s: Step %d: Train accuracy = %.1f%%' % (datetime.now(), i, train_accuracy * 100)) print('%s: Step %d: Cross entropy = %f' % (datetime.now(), i, cross_entropy_value)) validation_bottlenecks, validation_ground_truth = ( get_random_cached_bottlenecks( sess, image_lists, FLAGS.validation_batch_size, 'validation', FLAGS.bottleneck_dir, FLAGS.image_dir, jpeg_data_tensor, bottleneck_tensor)) validation_accuracy = sess.run( evaluation_step, feed_dict={bottleneck_input: validation_bottlenecks, ground_truth_input: validation_ground_truth}) print('%s: Step %d: Validation accuracy = %.1f%%' % (datetime.now(), i, validation_accuracy * 100)) # We've completed all our training, so run a final test evaluation on # some new images we haven't used before. test_bottlenecks, test_ground_truth = get_random_cached_bottlenecks( sess, image_lists, FLAGS.test_batch_size, 'testing', FLAGS.bottleneck_dir, FLAGS.image_dir, jpeg_data_tensor, bottleneck_tensor) test_accuracy = sess.run( evaluation_step, feed_dict={bottleneck_input: test_bottlenecks, ground_truth_input: test_ground_truth}) print('Final test accuracy = %.1f%%' % (test_accuracy * 100)) # Write out the trained graph and labels with the weights stored as constants. output_graph_def = graph_util.convert_variables_to_constants( sess, graph.as_graph_def(), [FLAGS.final_tensor_name]) with gfile.FastGFile(FLAGS.output_graph, 'wb') as f: f.write(output_graph_def.SerializeToString()) with gfile.FastGFile(FLAGS.output_labels, 'w') as f: f.write('\n'.join(image_lists.keys()) + '\n')
def learn(data_folder, experts, learning_rate=.001, train_ratio=.8, validation_ratio=.1, test_ratio=.1, save_every=10, batch_size=2048, hidden_size=1024, dropout=.5, epochs=500, print_every=1, model_dir='.', perceptron=False, mem_ratio=.95): assert train_ratio + validation_ratio + test_ratio == 1, 'Train/validation/test ratios must sum up to 1' data = read_data(data_folder, train_ratio, validation_ratio, test_ratio) model_name = ('''transfer_classifier_moe_epochs_{}_batch_{}_ratios_{}_{}_{}_''' '''learning_rate_{}'''.format( epochs, batch_size, train_ratio, validation_ratio, test_ratio, learning_rate)) if perceptron: model_name = '{}_perceptron.pb'.format(model_name) else: model_name = '{}_dropout_{}_hidden_size_{}.pb'.format(model_name, dropout, hidden_size) gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=mem_ratio) with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess: local_experts = {} for model in os.listdir(experts): print('Loading {}'.format(model)) load_graph(os.path.join(args.experts, model)) stripped = model[20:] h5 = stripped[:stripped.find('_')]# MESSY. local_experts[h5] = sess.graph.get_tensor_by_name('{}output:0'.format(h5)) data.train._X = np.hstack([ data.train.X, np.vstack([ flow(sess, local_experts, x) for x in chunks(data.train.X, batch_size) ]) ]) data.validation._X = np.hstack([ data.validation.X, flow(sess, local_experts, data.validation.X) ]) data.test._X = np.hstack([ data.test.X, flow(sess, local_experts, data.test.X) ]) x = tf.placeholder('float', shape=[None, data.train.X_features], name='input') y_ = tf.placeholder('float', shape=[None, data.train.Y_features], name='target') if perceptron: W = weight_variable([data.train.X_features, data.train.Y_features], name='weights') b = bias_variable([data.train.Y_features], name='bias') logits = tf.matmul(x,W) + b else: W_in = weight_variable([data.train.X_features, hidden_size], name='weights_in') b_in = bias_variable([hidden_size], name='bias_in') hidden = tf.matmul(x,W_in) + b_in relu = tf.nn.relu(hidden) keep_prob = tf.placeholder_with_default([1.], shape=None) hidden_dropout = tf.nn.dropout(relu, keep_prob) W_out = weight_variable([hidden_size,data.train.Y_features], name='weights_out') b_out = bias_variable([data.train.Y_features], name='bias_out') logits = tf.matmul(relu,W_out) + b_out y = tf.nn.softmax(logits, name='output') cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits, y_) train_step = tf.train.AdamOptimizer(learning_rate).minimize(cross_entropy) correct_prediction = tf.equal(tf.argmax(y,1), tf.argmax(y_,1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, 'float')) sess.run(tf.initialize_all_variables()) last_epoch = 0 t_epoch = time.time() while data.train.epoch <= epochs: epoch = data.train.epoch batch_x, batch_y = data.train.next_batch(batch_size) t_start = time.time() feed_dict = {x: batch_x, y_: batch_y } if perceptron else {x: batch_x, y_: batch_y, keep_prob: dropout} train_step.run(feed_dict=feed_dict) t_end = time.time() - t_start if epoch > last_epoch: if epoch % print_every == 0: train_accuracy = accuracy.eval(feed_dict={ x: batch_x, y_: batch_y }) validation_accuracy = accuracy.eval(feed_dict={ x: data.validation.X, y_: data.validation.Y }) print('''Epoch {} train accuracy: {}, validation accuracy: {}. ''' '''{} states/sec, {} secs/epoch.'''.format(epoch, train_accuracy, validation_accuracy, batch_size/t_end, time.time() - t_epoch)) if epoch % save_every == 0 or epoch == epochs: output_graph_def = graph_util.convert_variables_to_constants( sess, sess.graph.as_graph_def(), ['input', 'output']) with gfile.FastGFile(os.path.join(model_dir, model_name), 'w') as f: f.write(output_graph_def.SerializeToString()) t_epoch = time.time() last_epoch = epoch print('Trained model saved to {}'.format(os.path.join(model_dir, model_name))) if test_ratio > 0: test_accuracy = accuracy.eval(feed_dict={x: data.test.X, y_: data.test.Y }) print('Evaluation on testing data: {}'.format(test_accuracy))