def _to_tfrecord(image_ids_file, tfrecord_writer, source_dir):
with open(image_ids_file) as f:
img_ids = f.readlines()
num_images = len(img_ids)
with tf.Graph().as_default():
coder = dataset_utils.ImageCoder()
with tf.Session('') as sess:
for j in range(num_images):
xml_path = os.path.join(
source_dir, 'VOC2007/Annotations',
'{}.xml'.format(img_ids[j].strip('\n')))
img_path, label = _parse_xml(xml_path, source_dir)
sys.stdout.write('\r>> Reading file [%s] image %d/%d' %
(img_path, j + 1, num_images))
sys.stdout.flush()
# Get image, edge-map and cartooned image
img = misc.imread(img_path)
# Resize the image
h = np.size(img, 0)
w = np.size(img, 1)
# Encode the images
image_str = coder.encode_jpeg(img)
# Buil example
example = dataset_utils.image_to_tfexample(
image_str, 'jpg', h, w, label.tolist())
tfrecord_writer.write(example.SerializeToString())
def _convert_dataset(split_name, image_filenames):
assert split_name in ['train', 'validation', 'test']
num_per_shard = int(
math.ceil(len(image_filenames) / float(FLAGS.num_shards)))
with tf.Graph().as_default():
# image_reader = dataset_utils.ImageReader()
image_reader = dataset_utils.ImageCoder()
with tf.Session('') as sess:
for shard_id in range(FLAGS.num_shards):
output_filename = _get_dataset_filename(split_name, shard_id)
with tf.python_io.TFRecordWriter(
output_filename) as tfrecord_writer:
start_ndx = shard_id * num_per_shard
end_ndx = min((shard_id + 1) * num_per_shard,
len(image_filenames))
for i in range(start_ndx, end_ndx):
sys.stdout.write(
'\r>> Converting image %d/%d shard %d' %
(i + 1, len(image_filenames), shard_id))
sys.stdout.flush()
# read the image
img_filename = image_filenames[i]
img_data = tf.gfile.FastGFile(img_filename, 'r').read()
img_status, img_data, img_shape = image_reader.decode_image(
sess, img_data)
if img_status and img_data is not None:
example = dataset_utils.image_to_tfexample(
img_data, 'jpg', img_shape, img_filename)
tfrecord_writer.write(example.SerializeToString())
sys.stdout.write('\n')
sys.stdout.flush()
def _add_to_tfrecord(data_filename,
tfrecord_writer,
label_filename=None,
subset=None,
test_fold=False):
""" Loads data from the stl10 files and writes to a TFRecord.
Args:
data_filename: The filename of the stl10 file.
tfrecord_writer: The TFRecord writer to use for writing.
Returns:
The new offset.
"""
images = read_all_images(data_filename)
labels = None
if label_filename:
labels = read_labels(label_filename)
if subset:
if test_fold:
images = images[subset]
labels = labels[subset]
else:
images = np.delete(images, subset, axis=0)
labels = np.delete(labels, subset, axis=0)
num_images = images.shape[0]
with tf.Graph().as_default():
coder = dataset_utils.ImageCoder()
with tf.Session(''):
for j in range(num_images):
sys.stdout.write('\r>> Reading file [%s] image %d/%d' %
(data_filename, j + 1, num_images))
sys.stdout.flush()
# Get image, edge-map and cartooned image
image = np.squeeze(images[j])
if label_filename:
label = labels[j] - 1 # labels should be 0 indexed!
else:
label = -1
# Build example
img_str = coder.encode_jpeg(image)
example = dataset_utils.image_example(image, img_str,
int(label))
tfrecord_writer.write(example.SerializeToString())
return num_images
def to_tfrecord(image_ids_file, dest_dir, source_dir):
if not tf.gfile.Exists(dest_dir):
tf.gfile.MakeDirs(dest_dir)
with open(image_ids_file) as f:
img_ids = f.readlines()
num_images = len(img_ids)
num_per_class = {cn: 0 for cn in CLASSES}
writers = {
c: tf.python_io.TFRecordWriter(get_output_filename(dest_dir, c))
for c in CLASSES
}
with tf.Graph().as_default():
coder = dataset_utils.ImageCoder()
with tf.Session('') as sess:
for j in range(num_images):
# Parse the annotations file
mat_path = os.path.join(
source_dir, 'Annotations',
'{}.mat'.format(img_ids[j].strip('\n')))
examples = parse_mat(mat_path, source_dir)
for e in examples:
sys.stdout.write('\r>> Reading file [%s] image %d/%d' %
(e['image_path'], j + 1, num_images))
sys.stdout.flush()
# Get image, edge-map and cartooned image
img = misc.imread(e['image_path'], mode='RGB')
# Encode the images
image_str = coder.encode_jpeg(img)
# Build example
example = dataset_utils.to_tfexample(
image_str, 'jpg', e['im_size'].tolist(),
e['bbox'].tolist(), e['azimuth'], e['elevation'],
e['theta'])
# Write example
writers[e['class_name']].write(example.SerializeToString())
# Update number of examples per class
num_per_class[e['class_name']] += 1
print(num_per_class)
dataset_utils.save_obj(num_per_class, dest_dir, 'num_per_class')
def get_coder(self): """Gets an image encoder with the specified preprocessing function.""" return dataset_utils.ImageCoder(preprocess_fn=self.get_preprocessing_fn())