def tfrecord_matrix_id_loc_distinct( tfrecord_dir, size, channels): # write shards dynamically when called # similarly to tfrecord_matrix_id_loc, but with different id str. Checks if input_utils actually loads several tfrecords. tfrecord_loc = os.path.join(tfrecord_dir, 's28_matrix_float_0_128_256.tfrecord') if os.path.exists(tfrecord_loc): os.remove(tfrecord_loc) # monotonic labels, to check shuffling examples = [ { 'matrix': random_image(size, channels), 'id_str': str(n) } for n in range(128, 1024) ] # 3x the size of tfrecord_matrix_id_loc, for testing input read rates writer = tf.python_io.TFRecordWriter(tfrecord_loc) for example in examples: # depends on tfrecord.create_tfrecord writer.write( create_tfrecord.serialize_image_example(matrix=example['matrix'], id_str=example['id_str'])) writer.close() return tfrecord_loc
def test_serialize_image_example_extra_data(visual_check_image_data, extra_data_feature_spec): label = 1. an_int = 1 a_float = .5 some_floats = np.array([1., 2., 3.]) a_string = 'hello world' extra_kwargs = { 'an_int': 1, 'a_float': .5, 'some_floats': some_floats, 'a_string': a_string } serialized_example = create_tfrecord.serialize_image_example( visual_check_image_data, label=label, **extra_kwargs) with tf.Session() as sess: example = tf.parse_single_example(serialized_example, extra_data_feature_spec) recovered_matrix = tf.io.decode_raw(example['matrix'], out_type=tf.uint8).eval() assert np.allclose(recovered_matrix, visual_check_image_data.flatten()) assert example['label'].eval() == label assert example['an_int'].eval() == an_int assert np.isclose(example['a_float'].eval(), a_float) assert np.allclose(example['some_floats'].eval(), some_floats) assert example['a_string'].eval() == a_string.encode()
def example_tfrecords(tfrecord_train_loc, tfrecord_test_loc, example_data): tfrecord_locs = [tfrecord_train_loc, tfrecord_test_loc] for tfrecord_loc in tfrecord_locs: if os.path.exists(tfrecord_loc): os.remove(tfrecord_loc) writer = tf.python_io.TFRecordWriter(tfrecord_loc) for example in example_data: writer.write( create_tfrecord.serialize_image_example(matrix=example[0], label=example[1])) writer.close()
def test_serialize_image_example(visual_check_image_data, size, channels): serialized_example = create_tfrecord.serialize_image_example( visual_check_image_data, label=1.) # parse back and confirm it matches. Must be within session for tensors to be comparable to np with tf.Session() as sess: example = tf.parse_single_example( serialized_example, features=read_tfrecord.matrix_label_feature_spec(size, channels, float_label=True)) recovered_matrix = tf.io.decode_raw(example['matrix'], out_type=tf.uint8).eval() assert np.allclose(recovered_matrix, visual_check_image_data.flatten()) assert example['label'].eval() == 1.
def tfrecord_matrix_loc(tfrecord_dir, size, channels): # write shards dynamically when called tfrecord_loc = os.path.join(tfrecord_dir, 's28_matrix_0.tfrecord') if os.path.exists(tfrecord_loc): os.remove(tfrecord_loc) examples = [{'matrix': random_image(size, channels)} for n in range(128)] writer = tf.python_io.TFRecordWriter(tfrecord_loc) for example in examples: # depends on tfrecord.create_tfrecord writer.write( create_tfrecord.serialize_image_example(matrix=example['matrix'])) writer.close() return tfrecord_loc
def row_to_serialized_example(row, img_size, columns_to_save, reader): # row should have columns that exactly match a read_tfrecord feature spec function pil_img = reader(row) # pil_img.save('zoobot/test_examples/rescaled_after_pil.png') # to align with north/east # TODO refactor this to make sure it matches downloader final_pil_img = pil_img.resize(size=(img_size, img_size), resample=Image.LANCZOS).transpose( Image.FLIP_TOP_BOTTOM) matrix = np.array(final_pil_img) extra_kwargs = {} for col in columns_to_save: extra_kwargs.update({col: row[col]}) return create_tfrecord.serialize_image_example(matrix, **extra_kwargs)
def stratified_tfrecord_locs(tfrecord_dir, stratified_data): tfrecord_locs = [ os.path.join(tfrecord_dir, 'stratified_train.tfrecords'), os.path.join(tfrecord_dir, 'stratified_test.tfrecords') ] for tfrecord_loc in tfrecord_locs: if os.path.exists(tfrecord_loc): os.remove(tfrecord_loc) writer = tf.python_io.TFRecordWriter(tfrecord_loc) for example in stratified_data: # depends on tfrecord.create_tfrecord writer.write( create_tfrecord.serialize_image_example(matrix=example[0], label=example[1])) writer.close() return tfrecord_locs # of form [train_loc, test_loc]
def tfrecord_matrix_ints_loc(tfrecord_dir, size, channels): # write shards dynamically when called tfrecord_loc = os.path.join(tfrecord_dir, 's28_matrix_float_0.tfrecord') if os.path.exists(tfrecord_loc): os.remove(tfrecord_loc) # monotonic labels, to check shuffling examples = [{ 'matrix': random_image(size, channels), 'label': (n / 128.) } for n in range(128)] writer = tf.python_io.TFRecordWriter(tfrecord_loc) for example in examples: # depends on tfrecord.create_tfrecord writer.write( create_tfrecord.serialize_image_example(matrix=example['matrix'], label=example['label'])) writer.close() return tfrecord_loc
def shard_locs(tfrecord_dir, size, channels): # write shards dynamically when called shard_names = ['s28_shard_0.tfrecord', 's28_shard_1.tfrecord'] tfrecord_locs = list( map(lambda x: os.path.join(tfrecord_dir, x), shard_names)) for tfrecord_n, tfrecord_loc in enumerate(tfrecord_locs): if os.path.exists(tfrecord_loc): os.remove(tfrecord_loc) examples = [{ 'matrix': random_image(size, channels), 'id_str': str(tfrecord_n) + '_' + str(n) } for n in range(128)] writer = tf.python_io.TFRecordWriter(tfrecord_loc) for example in examples: # depends on tfrecord.create_tfrecord writer.write( create_tfrecord.serialize_image_example( matrix=example['matrix'], id_str=example['id_str'])) writer.close() return tfrecord_locs # of form [train_loc, test_loc]
def serialized_matrix_label_id_example(size, channels, unique_id): return create_tfrecord.serialize_image_example(matrix=random_image( size, channels), label=1., id_str=unique_id)
def serialized_matrix_label_example(size, channels): return create_tfrecord.serialize_image_example(matrix=random_image( size, channels), label=1.)