예제 #1
0
def _read_word_record(data_queue):

    reader = tf.TFRecordReader() # Construct a general reader
    key, example_serialized = reader.read(data_queue) 

    feature_map = {
        'image/encoded':  tf.FixedLenFeature( [], dtype=tf.string, 
                                              default_value='' ),
        'image/labels':   tf.VarLenFeature( dtype=tf.int64 ), 
        'image/width':    tf.FixedLenFeature( [1], dtype=tf.int64,
                                              default_value=1 ),
        'image/filename': tf.FixedLenFeature([], dtype=tf.string,
                                             default_value='' ),
        'text/string':     tf.FixedLenFeature([], dtype=tf.string,
                                             default_value='' ),
        'text/length':    tf.FixedLenFeature( [1], dtype=tf.int64,
                                              default_value=1 )
    }
    features = tf.parse_single_example( example_serialized, feature_map )

    image = tf.image.decode_jpeg( features['image/encoded'], channels=1 ) #gray
    width = tf.cast( features['image/width'], tf.int32) # for ctc_loss
    label = tf.serialize_sparse( features['image/labels'] ) # for batching
    length = features['text/length']
    text = features['text/string']
    filename = features['image/filename']
    return image,width,label,length,text,filename
  def testDeserializeFailsInconsistentRank(self):
    with self.test_session(use_gpu=False) as sess:
      sp_input0 = self._SparseTensorPlaceholder()
      sp_input1 = self._SparseTensorPlaceholder()
      input0_val = self._SparseTensorValue_5x6(np.arange(6))
      input1_val = self._SparseTensorValue_1x1x1()
      serialized0 = tf.serialize_sparse(sp_input0)
      serialized1 = tf.serialize_sparse(sp_input1)
      serialized_concat = tf.pack([serialized0, serialized1])

      sp_deserialized = tf.deserialize_many_sparse(
          serialized_concat, dtype=tf.int32)

      with self.assertRaisesOpError(
          r"Inconsistent rank across SparseTensors: rank prior to "
          r"SparseTensor\[1\] was: 3 but rank of SparseTensor\[1\] is: 4"):
        sess.run(
            sp_deserialized, {sp_input0: input0_val, sp_input1: input1_val})
  def testDeserializeFailsWrongType(self):
    with self.test_session(use_gpu=False) as sess:
      sp_input0 = self._SparseTensorPlaceholder()
      sp_input1 = self._SparseTensorPlaceholder()
      input0_val = self._SparseTensorValue_5x6(np.arange(6))
      input1_val = self._SparseTensorValue_3x4(np.arange(6))
      serialized0 = tf.serialize_sparse(sp_input0)
      serialized1 = tf.serialize_sparse(sp_input1)
      serialized_concat = tf.pack([serialized0, serialized1])

      sp_deserialized = tf.deserialize_many_sparse(
          serialized_concat, dtype=tf.int64)

      with self.assertRaisesOpError(
          r"Requested SparseTensor of type int64 but "
          r"SparseTensor\[0\].values.dtype\(\) == int32"):
        sess.run(
            sp_deserialized, {sp_input0: input0_val, sp_input1: input1_val})
예제 #4
0
  def testSerializeSparseTensor(self):
    sp_input = tf.SparseTensor(
        indices=tf.constant([[1]], dtype=tf.int64),
        values=tf.constant([2], dtype=tf.int64),
        dense_shape=[2])

    with self.cached_session():
      serialized_sp = tf.serialize_sparse(sp_input, 'serialize_name', tf.string)
      self.assertEqual((3,), serialized_sp.shape)
      self.assertTrue(serialized_sp[0].numpy())  # check non-empty
    def testDeserializeFailsWrongType(self):
        with self.test_session(use_gpu=False) as sess:
            sp_input0 = self._SparseTensorPlaceholder()
            sp_input1 = self._SparseTensorPlaceholder()
            input0_val = self._SparseTensorValue_5x6(np.arange(6))
            input1_val = self._SparseTensorValue_3x4(np.arange(6))
            serialized0 = tf.serialize_sparse(sp_input0)
            serialized1 = tf.serialize_sparse(sp_input1)
            serialized_concat = tf.pack([serialized0, serialized1])

            sp_deserialized = tf.deserialize_many_sparse(serialized_concat,
                                                         dtype=tf.int64)

            with self.assertRaisesOpError(
                    r"Requested SparseTensor of type int64 but "
                    r"SparseTensor\[0\].values.dtype\(\) == int32"):
                sess.run(sp_deserialized, {
                    sp_input0: input0_val,
                    sp_input1: input1_val
                })
  def testSerializeDeserializeMany(self):
    with self.test_session(use_gpu=False) as sess:
      sp_input0 = self._SparseTensorValue_5x6(np.arange(6))
      sp_input1 = self._SparseTensorValue_3x4(np.arange(6))
      serialized0 = tf.serialize_sparse(sp_input0)
      serialized1 = tf.serialize_sparse(sp_input1)
      serialized_concat = tf.pack([serialized0, serialized1])

      sp_deserialized = tf.deserialize_many_sparse(
          serialized_concat, dtype=tf.int32)

      combined_indices, combined_values, combined_shape = sess.run(
          sp_deserialized)

      self.assertAllEqual(combined_indices[:6, 0], [0] * 6)  # minibatch 0
      self.assertAllEqual(combined_indices[:6, 1:], sp_input0[0])
      self.assertAllEqual(combined_indices[6:, 0], [1] * 6)  # minibatch 1
      self.assertAllEqual(combined_indices[6:, 1:], sp_input1[0])
      self.assertAllEqual(combined_values[:6], sp_input0[1])
      self.assertAllEqual(combined_values[6:], sp_input1[1])
      self.assertAllEqual(combined_shape, [2, 5, 6])
    def testDeserializeFailsInconsistentRank(self):
        with self.test_session(use_gpu=False) as sess:
            sp_input0 = self._SparseTensorPlaceholder()
            sp_input1 = self._SparseTensorPlaceholder()
            input0_val = self._SparseTensorValue_5x6(np.arange(6))
            input1_val = self._SparseTensorValue_1x1x1()
            serialized0 = tf.serialize_sparse(sp_input0)
            serialized1 = tf.serialize_sparse(sp_input1)
            serialized_concat = tf.pack([serialized0, serialized1])

            sp_deserialized = tf.deserialize_many_sparse(serialized_concat,
                                                         dtype=tf.int32)

            with self.assertRaisesOpError(
                    r"Inconsistent rank across SparseTensors: rank prior to "
                    r"SparseTensor\[1\] was: 3 but rank of SparseTensor\[1\] is: 4"
            ):
                sess.run(sp_deserialized, {
                    sp_input0: input0_val,
                    sp_input1: input1_val
                })
  def testSerializeDeserializeMany(self):
    with self.test_session(use_gpu=False) as sess:
      sp_input0 = self._SparseTensorValue_5x6(np.arange(6))
      sp_input1 = self._SparseTensorValue_3x4(np.arange(6))
      serialized0 = tf.serialize_sparse(sp_input0)
      serialized1 = tf.serialize_sparse(sp_input1)
      serialized_concat = tf.stack([serialized0, serialized1])

      sp_deserialized = tf.deserialize_many_sparse(
          serialized_concat, dtype=tf.int32)

      combined_indices, combined_values, combined_shape = sess.run(
          sp_deserialized)

      self.assertAllEqual(combined_indices[:6, 0], [0] * 6)  # minibatch 0
      self.assertAllEqual(combined_indices[:6, 1:], sp_input0[0])
      self.assertAllEqual(combined_indices[6:, 0], [1] * 6)  # minibatch 1
      self.assertAllEqual(combined_indices[6:, 1:], sp_input1[0])
      self.assertAllEqual(combined_values[:6], sp_input0[1])
      self.assertAllEqual(combined_values[6:], sp_input1[1])
      self.assertAllEqual(combined_shape, [2, 5, 6])
예제 #9
0
def _ReadExamples(filename_queue, shape, using_ctc, reader=None):
    """Builds network input tensor ops for TF Example.

  Args:
    filename_queue: Queue of filenames, from tf.train_bkp.string_input_producer
    shape:          ImageShape with the desired shape of the input.
    using_ctc:      Take the unpadded_class labels instead of padded.
    reader:         Function that returns an actual reader to read Examples from
      input files. If None, uses tf.TFRecordReader().
  Returns:
    image:   Float Tensor containing the input image scaled to [-1.28, 1.27].
    height:  Tensor int64 containing the height of the image.
    width:   Tensor int64 containing the width of the image.
    labels:  Serialized SparseTensor containing the int64 labels.
    text:    Tensor string of the utf8 truth text.
  """
    if reader:
        reader = reader()
    else:
        reader = tf.TFRecordReader()
    _, example_serialized = reader.read(filename_queue)
    example_serialized = tf.reshape(example_serialized, shape=[])
    features = tf.parse_single_example(
        example_serialized, {
            'image/encoded':
            parsing_ops.FixedLenFeature([1], dtype=tf.string,
                                        default_value=''),
            'image/text':
            parsing_ops.FixedLenFeature([1], dtype=tf.string,
                                        default_value=''),
            'image/class':
            parsing_ops.VarLenFeature(dtype=tf.int64),
            'image/unpadded_class':
            parsing_ops.VarLenFeature(dtype=tf.int64),
            'image/height':
            parsing_ops.FixedLenFeature([1], dtype=tf.int64, default_value=1),
            'image/width':
            parsing_ops.FixedLenFeature([1], dtype=tf.int64, default_value=1)
        })
    if using_ctc:
        labels = features['image/unpadded_class']
    else:
        labels = features['image/class']
    labels = tf.serialize_sparse(labels)
    image = tf.reshape(features['image/encoded'], shape=[], name='encoded')
    image = _ImageProcessing(image, shape)
    height = tf.reshape(features['image/height'], [-1])
    width = tf.reshape(features['image/width'], [-1])
    text = tf.reshape(features['image/text'], shape=[])

    return image, height, width, labels, text
  def testDeserializeFailsInvalidProto(self):
    with self.test_session(use_gpu=False) as sess:
      sp_input0 = self._SparseTensorPlaceholder()
      input0_val = self._SparseTensorValue_5x6(np.arange(6))
      serialized0 = tf.serialize_sparse(sp_input0)
      serialized1 = ["a", "b", "c"]
      serialized_concat = tf.pack([serialized0, serialized1])

      sp_deserialized = tf.deserialize_many_sparse(
          serialized_concat, dtype=tf.int32)

      with self.assertRaisesOpError(
          r"Could not parse serialized_sparse\[1, 0\]"):
        sess.run(sp_deserialized, {sp_input0: input0_val})
    def testDeserializeFailsInvalidProto(self):
        with self.test_session(use_gpu=False) as sess:
            sp_input0 = self._SparseTensorPlaceholder()
            input0_val = self._SparseTensorValue_5x6(np.arange(6))
            serialized0 = tf.serialize_sparse(sp_input0)
            serialized1 = ["a", "b", "c"]
            serialized_concat = tf.pack([serialized0, serialized1])

            sp_deserialized = tf.deserialize_many_sparse(serialized_concat,
                                                         dtype=tf.int32)

            with self.assertRaisesOpError(
                    r"Could not parse serialized_sparse\[1, 0\]"):
                sess.run(sp_deserialized, {sp_input0: input0_val})
예제 #12
0
def _ReadExamples(filename_queue, shape, using_ctc, reader=None):
  """Builds network input tensor ops for TF Example.

  Args:
    filename_queue: Queue of filenames, from tf.train.string_input_producer
    shape:          ImageShape with the desired shape of the input.
    using_ctc:      Take the unpadded_class labels instead of padded.
    reader:         Function that returns an actual reader to read Examples from
      input files. If None, uses tf.TFRecordReader().
  Returns:
    image:   Float Tensor containing the input image scaled to [-1.28, 1.27].
    height:  Tensor int64 containing the height of the image.
    width:   Tensor int64 containing the width of the image.
    labels:  Serialized SparseTensor containing the int64 labels.
    text:    Tensor string of the utf8 truth text.
  """
  if reader:
    reader = reader()
  else:
    reader = tf.TFRecordReader()
  _, example_serialized = reader.read(filename_queue)
  example_serialized = tf.reshape(example_serialized, shape=[])
  features = tf.parse_single_example(
      example_serialized,
      {'image/encoded': parsing_ops.FixedLenFeature(
          [1], dtype=tf.string, default_value=''),
       'image/text': parsing_ops.FixedLenFeature(
           [1], dtype=tf.string, default_value=''),
       'image/class': parsing_ops.VarLenFeature(dtype=tf.int64),
       'image/unpadded_class': parsing_ops.VarLenFeature(dtype=tf.int64),
       'image/height': parsing_ops.FixedLenFeature(
           [1], dtype=tf.int64, default_value=1),
       'image/width': parsing_ops.FixedLenFeature(
           [1], dtype=tf.int64, default_value=1)})
  if using_ctc:
    labels = features['image/unpadded_class']
  else:
    labels = features['image/class']
  labels = tf.serialize_sparse(labels)
  image = tf.reshape(features['image/encoded'], shape=[], name='encoded')
  image = _ImageProcessing(image, shape)
  height = tf.reshape(features['image/height'], [-1])
  width = tf.reshape(features['image/width'], [-1])
  text = tf.reshape(features['image/text'], shape=[])

  return image, height, width, labels, text
예제 #13
0
def batchReadTf(tfpath):

    with tf.Session() as sess:
        # Create a list of filenames and pass it to a queue
        filename_queue = tf.train.string_input_producer([tfpath], num_epochs=1)
        # Define a reader and read the next record
        reader = tf.TFRecordReader()
        _, serialized_example = reader.read(filename_queue)
        # Decode the record read by the reader
        features = tf.parse_single_example(serialized_example, features=feature_map)

        image = tf.image.decode_jpeg(features['image/encoded'], channels=1)  # gray
        width = tf.cast(features['image/width'], tf.int32)  # for ctc_loss
        label = tf.serialize_sparse(features['image/labels'])  # for batching
        length = features['text/length']
        text = features['text/string']
        filename = features['image/filename']

        # print text

        init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer())
        sess.run(init_op)
        # Create a coordinator and run all QueueRunner objects
        coord = tf.train.Coordinator()
        threads = tf.train.start_queue_runners(coord=coord)
        for batch_index in range(20):
            img, lbl = sess.run([image, text])
            print lbl,img.shape
            img = img.reshape((img.shape[0:2]))
            pimg = Image.fromarray(img, "L")

            # pimg.save("../img/%d-%s.jpg" % (batch_index,lbl))
            # img = img.astype(np.uint8)
            # for j in range(6):
            #     plt.subplot(2, 3, j + 1)
            #     plt.imshow(img[j, ...])        example.features.feature['image/encoded']

            #     plt.title('cat' if lbl[j] == 0 else 'dog')
            # plt.show()
        # Stop the threads
        coord.request_stop()

        # Wait for threads to stop
        coord.join(threads)

        sess.close()
예제 #14
0
    def _parser(self, serialized, distort=False):
        tensor_dict = ops.parse_sequence_example(serialized)
        tensor_dict['image/data'] = self._preproc(tensor_dict['image/data'])
        (tensor_dict['image/data'],
         tensor_dict['image/seq/char/bbox'],
         tensor_dict['image/seq/line/bbox']) = preproc.pad_borders_or_shrink(
            tensor_dict['image/data'], tensor_dict['image/seq/char/bbox'],
            tensor_dict['image/seq/line/bbox'], self.pad_shape)

        if self.sparse_labels:
            tensor_dict['image/seq/char/id_sparse'] = tf.serialize_sparse(
                ops.sparsify_label(tensor_dict['image/seq/char/id'],
                                   tensor_dict['image/char/count'])
            )

        # if distort: image = distort_image(image)
        self.feat_keys, features = zip(*tensor_dict.items())
        return features
예제 #15
0
    def txt_parse_fn(self, line, dataset_name):
        """
        """
        line = tf.strings.split([line], sep='\t', maxsplit=1)
        line = tf.sparse.to_dense(line, default_value='')

        filename = tf.reshape(line[0][0], [])
        text = tf.reshape(line[0][1], [])

        length = tf.reshape(tf.py_func(text_length, [text], tf.int64), [-1])

        text = tf.py_func(self.join_with_delimiter, [text], tf.string)
        text = tf.strings.join([text, EOS_TOKEN], separator='')

        image_content = tf.read_file(filename)
        image = tf.image.decode_image(image_content,
                                      channels=self.img_channels)
        image.set_shape([None, None, self.img_channels])

        label = self.indexing_fn(text, length, DELIMITER)
        label = tf.serialize_sparse(label)

        return image, label, length, text, filename, dataset_name
예제 #16
0
 def serialize_sparse_tensor(self, sparse_tensor):
     return tf.serialize_tensor(tf.serialize_sparse(sparse_tensor)).numpy()