def test_get_input_fn(self):
options_str = r"""
vcr_reader {
input_pattern: "output/uncased/VCR-RAW/val.record-00000-of-00005"
shuffle_buffer_size: 10
interleave_cycle_length: 1
batch_size: 8
prefetch_buffer_size: 8000
desired_size: 600
vocab_file: "data/bert/tf1.x/BERT-Base/vocab.txt"
out_of_vocabulary_token_id: 100
}
"""
options = text_format.Merge(options_str, reader_pb2.Reader())
dataset = reader.get_input_fn(options, is_training=False)()
for elem in dataset.take(1):
import pdb
pdb.set_trace()
for key, value in elem.items():
logging.info('%s: %s', key, value.shape)
logging.info('Examples:')
logging.info('answer_choices: %s', elem['answer_choices'][0])
logging.info('answer_choices_tag: %s',
elem['answer_choices_tag'][0])
logging.info('answer_choices_len: %s',
elem['answer_choices_len'][0])
logging.info('rationale_choices: %s', elem['rationale_choices'][0])
logging.info('rationale_choices_tag: %s',
elem['rationale_choices_tag'][0])
logging.info('rationale_choices_len: %s',
elem['rationale_choices_len'][0])
def test_get_input_fn(self):
options_str = r"""
vcr_reader {
input_pattern: "/own_files/yekeren/VCR-2stages-allboxes/val.record-*-of-00005"
shuffle_buffer_size: 10
interleave_cycle_length: 1
batch_size: 20
prefetch_buffer_size: 8000
frcnn_feature_dims: 1536
bert_feature_dims: 768
decode_jpeg: false
}
"""
options = text_format.Merge(options_str, reader_pb2.Reader())
dataset = reader.get_input_fn(options, is_training=False)()
for elem in dataset.take(1):
self.assertNotIn(InputFields.img_data, elem)
self.assertNotIn(InputFields.img_width, elem)
self.assertNotIn(InputFields.img_height, elem)
(num_objects, object_bboxes, object_labels, object_scores,
object_features) = (elem[InputFields.num_objects],
elem[InputFields.object_bboxes],
elem[InputFields.object_labels],
elem[InputFields.object_scores],
elem[InputFields.object_features])
import pdb
pdb.set_trace()
for key, value in elem.items():
logging.info('=' * 64)
logging.info(key)
logging.info(value)
def test_get_examples(self):
tf.reset_default_graph()
input_fn = reader.get_input_fn(self._options)
dataset = input_fn()
iterator = dataset.make_initializable_iterator()
feature_dict = iterator.get_next()
with self.test_session() as sess:
sess.run(iterator.initializer)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
try:
i = 0
while not coord.should_stop():
feature_values = sess.run(feature_dict)
self.assertAllEqual(
feature_values[InputDataFields.image].shape,
[7, 448, 448, 3])
self.assertAllEqual(
feature_values[InputDataFields.image_id].shape, [7])
self.assertAllEqual(
feature_values[InputDataFields.num_captions].shape,
[7])
self.assertEqual(
len(feature_values[
InputDataFields.caption_lengths].shape), 2)
self.assertEqual(
len(feature_values[
InputDataFields.caption_strings].shape), 3)
self.assertEqual(
feature_values[
InputDataFields.caption_lengths].shape[0], 7)
self.assertEqual(
feature_values[
InputDataFields.caption_strings].shape[0], 7)
self.assertEqual(
feature_values[
InputDataFields.caption_strings].shape[1],
feature_values[
InputDataFields.caption_lengths].shape[1])
i += 1
if i >= 10:
coord.request_stop()
except tf.errors.OutOfRangeError:
tf.logging.info("OutOfRangeError, done!")
finally:
tf.logging.info("Request to stop!")
coord.request_stop()
coord.join(threads)
def test_get_input_fn(self):
batch_size = 17
# input_pattern: "data-mscoco/tfrecords_v2_prop50_iou/coco_sgs.tfreocrd-00000-of-00020"
options_str = r"""
caption_graph_reader {
input_pattern: "/own_files/yekeren/WSSGG/data-vspnet/tfrecords/caption-graph-hanwang-v1/train.tfrecord*"
batch_size: %i
shuffle_buffer_size: 500
prefetch_buffer_size: 500
feature_dimensions: 1536
max_n_proposal: 20
}
""" % (batch_size)
options = text_format.Merge(options_str, reader_pb2.Reader())
dataset = reader.get_input_fn(options, is_training=False)()
for elem in dataset.take(1):
import pdb
pdb.set_trace()
self.assertAllEqual(elem['id'].shape, [batch_size])
self.assertAllEqual(elem['image/n_proposal'].shape, [batch_size])
self.assertAllEqual(elem['image/proposal'].shape,
[batch_size, 20, 4])
self.assertAllEqual(elem['image/proposal/feature'].shape,
[batch_size, 20, 1536])
self.assertDTypeEqual(elem['id'], np.int64)
self.assertDTypeEqual(elem['image/n_proposal'], np.int32)
self.assertDTypeEqual(elem['image/proposal'], np.float32)
self.assertDTypeEqual(elem['image/proposal/feature'], np.float32)
# Check graphs.
for scope in ['caption_graph']:
self.assertDTypeEqual(elem[scope + '/caption'], np.object)
max_n_node = elem[scope + '/n_node'].numpy().max()
max_n_edge = elem[scope + '/n_edge'].numpy().max()
self.assertAllEqual(elem[scope + '/nodes'].shape,
[batch_size, max_n_node])
self.assertAllEqual(elem[scope + '/edges'].shape,
[batch_size, max_n_edge])
self.assertAllEqual(elem[scope + '/senders'].shape,
[batch_size, max_n_edge])
self.assertAllEqual(elem[scope + '/receivers'].shape,
[batch_size, max_n_edge])
self.assertDTypeEqual(elem[scope + '/n_node'], np.int32)
self.assertDTypeEqual(elem[scope + '/n_edge'], np.int32)
self.assertDTypeEqual(elem[scope + '/nodes'], np.object)
self.assertDTypeEqual(elem[scope + '/edges'], np.object)
self.assertDTypeEqual(elem[scope + '/senders'], np.int32)
self.assertDTypeEqual(elem[scope + '/receivers'], np.int32)
def test_get_input_fn(self):
options_str = r"""
vcr_text_frcnn_reader {
input_pattern: "output/uncased/VCR-text_and_frcnn/val.record-*-of-00005"
shuffle_buffer_size: 10
interleave_cycle_length: 1
batch_size: 60
prefetch_buffer_size: 8000
frcnn_feature_dims: 1536
num_parallel_calls: 10
}
"""
options = text_format.Merge(options_str, reader_pb2.Reader())
dataset = reader.get_input_fn(options, is_training=False)()
for elem in dataset.take(10000):
for key, value in elem.items():
import pdb
pdb.set_trace()
j = 1
def test_get_input_fn(self):
options_str = r"""
vcr_text_only_reader {
input_pattern: "output/uncased/VCR-text_only/val.record-*-of-00005"
shuffle_buffer_size: 10
interleave_cycle_length: 1
batch_size: 3
prefetch_buffer_size: 8000
}
"""
options = text_format.Merge(options_str, reader_pb2.Reader())
dataset = reader.get_input_fn(options, is_training=False)()
for elem in dataset.take(1):
for key, value in elem.items():
logging.info('=' * 64)
logging.info(key)
logging.info(value)
import pdb
pdb.set_trace()
j = 1