def main(unused_argv): with tf.device("/cpu:0"): image = np.random.uniform(-1, 1, [1, 300, 300, 3]) g = tf.Graph() with g.as_default(): feature_extractor = FeatureExtractor() inputs = tf.placeholder(tf.float32, shape=image.shape) features = feature_extractor(inputs) with tf.Session(graph=g) as sess: saver = tf.train.Saver(var_list=feature_extractor.trainable_variables) saver.restore(sess, get_resnet_v2_101_checkpoint()) results = sess.run(features, feed_dict={inputs: image}) assert(all([x == y for x, y in zip(results.shape, [1, 1, 1, 2048])])) tf.logging.info("Successfully passed test.")
def main(unused_argv): image = load_image_from_path("images/image.jpg")[np.newaxis, ...] vocab, pretrained_matrix = load_glove(vocab_size=100, embedding_size=50) pos, pos_embeddings = get_parts_of_speech(), np.random.normal( 0, 0.1, [15, 50]) with tf.Graph().as_default(): inputs = tf.placeholder(tf.float32, shape=image.shape) box_extractor = BoxExtractor(get_faster_rcnn_config(), top_k_boxes=16) boxes, scores, cropped_inputs = box_extractor(inputs) feature_extractor = FeatureExtractor() mean_image_features = tf.reduce_mean(feature_extractor(inputs), [1, 2]) mean_object_features = tf.reshape( tf.reduce_mean(feature_extractor(cropped_inputs), [1, 2]), [1, 16, 2048]) image_captioner = PartOfSpeechImageCaptioner(UpDownCell(50), vocab, pretrained_matrix, UpDownCell(50), UpDownCell(50), pos, pos_embeddings) pos_logits, pos_logits_ids, word_logits, word_logits_ids = image_captioner( mean_image_features=mean_image_features, mean_object_features=mean_object_features) with tf.Session() as sess: box_saver = tf.train.Saver(var_list=box_extractor.variables) resnet_saver = tf.train.Saver(var_list=feature_extractor.variables) box_saver.restore(sess, get_faster_rcnn_checkpoint()) resnet_saver.restore(sess, get_resnet_v2_101_checkpoint()) sess.run(tf.variables_initializer(image_captioner.variables)) results = sess.run( [pos_logits, pos_logits_ids, word_logits, word_logits_ids], feed_dict={inputs: image}) assert (results[2].shape[0] == 1 and results[2].shape[1] == 3 and results[2].shape[3] == 100) tf.logging.info("Successfully passed test.")
def __init__(self): """Creates handles to the TensorFlow computational graph.""" # TensorFlow ops for JPEG decoding. self.encoded_jpeg = tf.placeholder(dtype=tf.string) self.decoded_jpeg = tf.image.decode_jpeg(self.encoded_jpeg, channels=3) self.decoded_jpeg = tf.image.resize_images( self.decoded_jpeg, [FLAGS.image_height, FLAGS.image_width]) # Create the model to extract image boxes self.box_extractor = BoxExtractor(get_faster_rcnn_config(), trainable=False) self.image_tensor = tf.placeholder( tf.float32, name='image_tensor', shape=[None, FLAGS.image_height, FLAGS.image_width, 3]) self.boxes, self.scores, self.cropped_images = self.box_extractor( self.image_tensor) # Create the model to extract the image features self.feature_extractor = FeatureExtractor(is_training=False, global_pool=False) # Compute the mean ResNet-101 features self.image_features = tf.reduce_mean( self.feature_extractor(self.image_tensor), [1, 2]) feature_batch = tf.shape(self.image_features)[0] feature_depth = tf.shape(self.image_features)[1] self.object_features = tf.reduce_mean( self.feature_extractor(self.cropped_images), [1, 2]) self.object_features = tf.reshape(self.object_features, [feature_batch, 100, feature_depth]) # Create a single TensorFlow Session for all image decoding calls. self.sess = tf.Session() rcnn_saver = tf.train.Saver(var_list=self.box_extractor.variables) resnet_saver = tf.train.Saver( var_list=self.feature_extractor.variables) rcnn_saver.restore(self.sess, get_faster_rcnn_checkpoint()) resnet_saver.restore(self.sess, get_resnet_v2_101_checkpoint()) self.lock = threading.Lock() self.attribute_map = get_visual_attributes()
def main(unused_argv): def _is_valid_num_shards(num_shards): """Returns True if num_shards is compatible with FLAGS.num_threads.""" return num_shards < FLAGS.num_threads or not num_shards % FLAGS.num_threads assert _is_valid_num_shards(FLAGS.train_shards), ( "Please make the FLAGS.num_threads commensurate with FLAGS.train_shards") assert _is_valid_num_shards(FLAGS.val_shards), ( "Please make the FLAGS.num_threads commensurate with FLAGS.val_shards") assert _is_valid_num_shards(FLAGS.test_shards), ( "Please make the FLAGS.num_threads commensurate with FLAGS.test_shards") # Create vocabulary from the glove embeddings. vocab, _ = load_glove(vocab_size=FLAGS.vocab_size, embedding_size=FLAGS.embedding_size) tagger = load_tagger() if not tf.gfile.IsDirectory(FLAGS.output_dir): tf.gfile.MakeDirs(FLAGS.output_dir) # Load image metadata from caption files. mscoco_train_dataset = _load_and_process_metadata(FLAGS.train_captions_file, FLAGS.train_image_dir) mscoco_val_dataset = _load_and_process_metadata(FLAGS.val_captions_file, FLAGS.val_image_dir) # Redistribute the MSCOCO data as follows: # train_dataset = 99% of mscoco_train_dataset # val_dataset = 1% of mscoco_train_dataset (for validation during training). # test_dataset = 100% of mscoco_val_dataset (for final evaluation). train_cutoff = int(0.99 * len(mscoco_train_dataset)) train_dataset = mscoco_train_dataset[:train_cutoff] val_dataset = mscoco_train_dataset[train_cutoff:] test_dataset = mscoco_val_dataset # If needed crop the dataset to make it smaller max_train_size = len(train_dataset) if FLAGS.train_dataset_size < max_train_size: # Shuffle the ordering of images. Make the randomization repeatable. random.seed(12345) random.shuffle(train_dataset) train_dataset = train_dataset[:FLAGS.train_dataset_size] max_val_size = len(val_dataset) if FLAGS.val_dataset_size < max_val_size: # Shuffle the ordering of images. Make the randomization repeatable. random.seed(12345) random.shuffle(val_dataset) val_dataset = val_dataset[:FLAGS.val_dataset_size] max_test_size = len(test_dataset) if FLAGS.test_dataset_size < max_test_size: # Shuffle the ordering of images. Make the randomization repeatable. random.seed(12345) random.shuffle(test_dataset) test_dataset = test_dataset[:FLAGS.test_dataset_size] # Create the model to extract image boxes box_extractor = BoxExtractor(get_faster_rcnn_config(), top_k_boxes=FLAGS.top_k_boxes, trainable=False) image_tensor = tf.placeholder(tf.float32, name='image_tensor', shape=[None, FLAGS.image_height, FLAGS.image_width, 3]) boxes, scores, cropped_images = box_extractor(image_tensor) # Create the model to extract the image features feature_extractor = FeatureExtractor(is_training=False, global_pool=False) # Compute the ResNet-101 features image_features = feature_extractor(image_tensor) feature_batch = tf.shape(image_features)[0] feature_depth = tf.shape(image_features)[3] object_features = tf.reduce_mean(feature_extractor(cropped_images), [1, 2]) object_features = tf.reshape(object_features, [feature_batch, FLAGS.top_k_boxes, feature_depth]) with tf.Session() as sess: rcnn_saver = tf.train.Saver(var_list=box_extractor.variables) resnet_saver = tf.train.Saver(var_list=feature_extractor.variables) rcnn_saver.restore(sess, get_faster_rcnn_checkpoint()) resnet_saver.restore(sess, get_resnet_v2_101_checkpoint()) lock = threading.Lock() def run_model_fn(images): lock.acquire() r = sess.run([image_features, object_features], feed_dict={image_tensor: images}) lock.release() return r _process_dataset("train", train_dataset, vocab, tagger, FLAGS.train_shards, run_model_fn) _process_dataset("val", val_dataset, vocab, tagger, FLAGS.val_shards, run_model_fn) _process_dataset("test", test_dataset, vocab, tagger, FLAGS.test_shards, run_model_fn)
batch_size = tf.shape(image_features)[0] num_boxes = tf.shape(object_features)[0] // batch_size depth = tf.shape(image_features)[1] object_features = tf.reshape(object_features, [batch_size, num_boxes, depth]) logits, image_detections, object_detections = attribute_detector( image_features, object_features) saver = tf.train.Saver(var_list=box_extractor.variables) feature_extractor_saver = tf.train.Saver( var_list=feature_extractor.variables) attribute_detector_saver = tf.train.Saver( var_list=attribute_detector.variables) faster_rcnn_ckpt = get_faster_rcnn_checkpoint() feature_extractor_ckpt = get_resnet_v2_101_checkpoint() attribute_detector_ckpt, attribute_detector_ckpt_name = get_attribute_detector_checkpoint( ) with tf.Session() as sess: assert (feature_extractor_ckpt is not None and attribute_detector_ckpt is not None) saver.restore(sess, faster_rcnn_ckpt) feature_extractor_saver.restore(sess, feature_extractor_ckpt) attribute_detector_saver.restore(sess, attribute_detector_ckpt) for i, (name, image) in enumerate(zip(list_of_filenames, list_of_images)):