def _convert_dataset(dataset_split):
"""Converts the specified dataset split to TFRecord format.
Args:
dataset_split: The dataset split (e.g., train, val).
Raises:
RuntimeError: If loaded image and label have different shape, or if the
image file with specified postfix could not be found.
"""
image_files = _get_files('image', dataset_split)
label_files = _get_files('label', dataset_split)
num_images = len(image_files)
num_labels = len(label_files)
num_per_shard = int(math.ceil(num_images / _NUM_SHARDS))
if num_images != num_labels:
raise RuntimeError(
"The number of images and labels doesn't match: {} {}".format(
num_images, num_labels))
image_reader = build_data.ImageReader('png', channels=3)
label_reader = build_data.ImageReader('png', channels=1)
for shard_id in range(_NUM_SHARDS):
shard_filename = '%s_fine-%05d-of-%05d.tfrecord' % (
dataset_split, shard_id, _NUM_SHARDS)
output_filename = os.path.join(FLAGS.output_dir, shard_filename)
with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer:
start_idx = shard_id * num_per_shard
end_idx = min((shard_id + 1) * num_per_shard, num_images)
for i in range(start_idx, end_idx):
sys.stdout.write('\r>> Converting image %d/%d shard %d' %
(i + 1, num_images, shard_id))
sys.stdout.flush()
# Read the image.
image_data = tf.gfile.FastGFile(image_files[i], 'rb').read()
height, width = image_reader.read_image_dims(image_data)
# Read the semantic segmentation annotation.
seg_data = tf.gfile.FastGFile(label_files[i], 'rb').read()
seg_height, seg_width = label_reader.read_image_dims(seg_data)
if height != seg_height or width != seg_width:
raise RuntimeError(
'Shape mismatched between image and label.')
# Convert to tf example.
re_match = _IMAGE_FILENAME_RE.search(image_files[i])
if re_match is None:
raise RuntimeError('Invalid image filename: ' +
image_files[i])
filename = os.path.basename(re_match.group(1))
example = build_data.image_seg_to_tfexample(
image_data, filename, height, width, seg_data)
tfrecord_writer.write(example.SerializeToString())
sys.stdout.write('\n')
sys.stdout.flush()
def _convert_dataset(dataset_split, dataset_dir, dataset_label_dir):
""" Converts the ADE20k dataset into into tfrecord format (SSTable).
Args:
dataset_split: Dataset split (e.g., train, val).
dataset_dir: Dir in which the dataset locates.
dataset_label_dir: Dir in which the annotations locates.
Raises:
RuntimeError: If loaded image and label have different shape.
"""
img_names = tf.gfile.Glob(os.path.join(dataset_dir, '*.jpg'))
random.shuffle(img_names)
seg_names = []
for f in img_names:
# get the filename without the extension
basename = os.path.basename(f).split(".")[0]
# cover its corresponding *_seg.png
seg = os.path.join(dataset_label_dir, basename + '.png')
seg_names.append(seg)
num_images = len(img_names)
num_per_shard = int(math.ceil(num_images / float(_NUM_SHARDS)))
image_reader = build_data.ImageReader('jpeg', channels=3)
label_reader = build_data.ImageReader('png', channels=1)
for shard_id in range(_NUM_SHARDS):
output_filename = os.path.join(
FLAGS.output_dir, '%s-%05d-of-%05d.tfrecord' %
(dataset_split, shard_id, _NUM_SHARDS))
with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer:
start_idx = shard_id * num_per_shard
end_idx = min((shard_id + 1) * num_per_shard, num_images)
for i in range(start_idx, end_idx):
sys.stdout.write('\r>> Converting image %d/%d shard %d' %
(i + 1, num_images, shard_id))
sys.stdout.flush()
# Read the image.
image_filename = img_names[i]
image_data = tf.gfile.FastGFile(image_filename, 'rb').read()
height, width = image_reader.read_image_dims(image_data)
# Read the semantic segmentation annotation.
seg_filename = seg_names[i]
seg_data = tf.gfile.FastGFile(seg_filename, 'rb').read()
seg_height, seg_width = label_reader.read_image_dims(seg_data)
if height != seg_height or width != seg_width:
raise RuntimeError(
'Shape mismatched between image and label.')
# Convert to tf example.
example = build_data.image_seg_to_tfexample(
image_data, img_names[i], height, width, seg_data)
tfrecord_writer.write(example.SerializeToString())
sys.stdout.write('\n')
sys.stdout.flush()
def _convert_dataset(dataset_split):
"""Converts the specified dataset split to TFRecord format.
Args:
dataset_split: The dataset split (e.g., train, test).
Raises:
RuntimeError: If loaded image and label have different shape.
"""
dataset = os.path.basename(dataset_split)[:-4]
sys.stdout.write('Processing ' + dataset)
filenames = [x.strip('\n') for x in open(dataset_split, 'r')]
num_images = len(filenames)
num_per_shard = int(math.ceil(num_images / _NUM_SHARDS))
image_reader = build_data.ImageReader('jpeg', channels=3)
label_reader = build_data.ImageReader('jpeg', channels=1)
for shard_id in range(_NUM_SHARDS):
output_filename = os.path.join(
FLAGS.output_dir,
'%s-%05d-of-%05d.tfrecord' % (dataset, shard_id, _NUM_SHARDS))
with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer:
start_idx = shard_id * num_per_shard
end_idx = min((shard_id + 1) * num_per_shard, num_images)
for i in range(start_idx, end_idx):
sys.stdout.write('\r>> Converting image %d/%d shard %d' %
(i + 1, len(filenames), shard_id))
sys.stdout.flush()
# Read the image.
image_filename = os.path.join(
#MH:
#FLAGS.image_folder, filenames[i] + '.' + FLAGS.image_format)
FLAGS.image_folder,
filenames[i] + '.jpg')
#END MH
image_data = tf.gfile.GFile(image_filename, 'rb').read()
height, width = image_reader.read_image_dims(image_data)
# Read the semantic segmentation annotation.
seg_filename = os.path.join(
FLAGS.semantic_segmentation_folder,
#MH:
#filenames[i] + '.' + FLAGS.label_format)
filenames[i] + '.jpg')
#END MH
seg_data = tf.gfile.GFile(seg_filename, 'rb').read()
seg_height, seg_width = label_reader.read_image_dims(seg_data)
if height != seg_height or width != seg_width:
raise RuntimeError(
'Shape mismatched between image and label.')
# Convert to tf example.
example = build_data.image_seg_to_tfexample(
image_data, filenames[i], height, width, seg_data)
tfrecord_writer.write(example.SerializeToString())
sys.stdout.write('\n')
sys.stdout.flush()
def create_tfrecords(src_files, file_ids, n_shards, sub_seq, use_tif,
output_dir):
if use_tif:
image_reader = build_data.ImageReader('png', channels=1)
else:
image_reader = build_data.ImageReader('jpeg', channels=1)
label_reader = build_data.ImageReader('png', channels=1)
n_images = len(src_files)
n_per_shard = int(math.ceil(n_images / float(n_shards)))
os.makedirs(output_dir, exist_ok=True)
print('Creating {} shards with {} images ({} per shard)'.format(
n_shards, n_images, n_per_shard))
for shard_id in range(n_shards):
output_file_path = os.path.join(
output_dir,
'{:s}-{:05d}-of-{:05d}.tfrecord'.format(sub_seq, shard_id,
n_shards))
with tf.python_io.TFRecordWriter(output_file_path) as tfrecord_writer:
start_idx = shard_id * n_per_shard
end_idx = min((shard_id + 1) * n_per_shard, n_images)
for img_id in tqdm(range(start_idx, end_idx), ncols=50):
img_src_file = src_files[img_id]
img_src_file_id, seq_name, seg_src_path, img_src_path = file_ids[
img_src_file]
image_data = tf.gfile.FastGFile(img_src_path, 'rb').read()
height, width = image_reader.read_image_dims(image_data)
if seg_src_path is not None:
seg_data = tf.gfile.FastGFile(seg_src_path, 'rb').read()
seg_height, seg_width = label_reader.read_image_dims(
seg_data)
if height != seg_height or width != seg_width:
raise RuntimeError(
'Shape mismatch found between image and label')
else:
seg_data = None
# Convert to tf example.
example = build_data.image_seg_to_tfexample(
image_data, img_src_path, height, width, seg_data)
tfrecord_writer.write(example.SerializeToString())
def _convert_dataset(dataset_split):
dataset = os.path.basename(dataset_split)[:-4]
sys.stdout.write('Processing ' + dataset)
filenames = [x.strip('\n') for x in open(dataset_split, 'r')]
print(filenames)
num_images = len(filenames)
num_per_shard = int(math.ceil(num_images / float(_NUM_SHARDS)))
image_reader = build_data.ImageReader('jpeg', channels=3)
label_reader = build_data.ImageReader('jpg', channels=1)
for shard_id in range(_NUM_SHARDS):
output_filename = os.path.join(
FLAGS.output_dir,
'%s-%05d-of-%05d.tfrecord' % (dataset, shard_id, _NUM_SHARDS))
with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer:
start_idx = shard_id * num_per_shard
end_idx = min((shard_id + 1) * num_per_shard, num_images)
for i in range(start_idx, end_idx):
sys.stdout.write('\r>> Converting image %d/%d shard %d' % (
i + 1, len(filenames), shard_id))
print("hell")
sys.stdout.flush()
# Read the image.
image_filename = os.path.join(
FLAGS.image_folder, filenames[i])# + '.' + FLAGS.image_format)
print(image_filename)
image_data = tf.gfile.GFile(image_filename, 'rb').read()
print("what")
height, width = image_reader.read_image_dims(image_data)
print("the")
# Read the semantic segmentation annotation.
seg_filename = os.path.join(
FLAGS.semantic_segmentation_folder,
filenames[i])# + '.' + FLAGS.label_format)
seg_data = tf.gfile.GFile(seg_filename, 'rb').read()
seg_height, seg_width = label_reader.read_image_dims(seg_data)
if height != seg_height or width != seg_width:
raise RuntimeError('Shape mismatched between image and label.')
# Convert to tf example.
example = build_data.image_seg_to_tfexample(
image_data, filenames[i], height, width, seg_data)
B = tf.io.TFRecordWriter('./tfrecord/storage.tfrecord')
B.write(example.SerializeToString())
print("me")
sys.stdout.write('\n')
sys.stdout.flush()
def _convert_dataset(dataset_split):
"""Converts the specified dataset split to TFRecord format.
Args:
dataset_split: The dataset split (e.g., train, test).
Raises:
RuntimeError: If loaded image and label have different shape.
"""
# val.txt
dataset = os.path.basename(dataset_split)[:-4]
input('dataset: %s' % dataset)
sys.stdout.write('Processing ' + dataset)
input('after processing')
filenames = [x.strip('\n') for x in open(dataset_split, 'r')]
num_images = len(filenames)
num_per_shard = int(math.ceil(num_images / float(_NUM_SHARDS)))
input('before image reader')
image_reader = build_data.ImageReader('jpeg', channels=3)
# label_reader = build_data.ImageReader('png', channels=1)
label_reader = build_data.ImageReader('jpeg', channels=3)
for shard_id in range(_NUM_SHARDS):
output_filename = os.path.join(
FLAGS.output_dir,
'%s-%05d-of-%05d.tfrecord' % (dataset, shard_id, _NUM_SHARDS))
input('before writer')
with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer:
start_idx = shard_id * num_per_shard
end_idx = min((shard_id + 1) * num_per_shard, num_images)
for i in range(start_idx, end_idx):
sys.stdout.write('\r>> Converting image %d/%d shard %d' % (
i + 1, len(filenames), shard_id))
sys.stdout.flush()
# Read the image.
image_filename = os.path.join(
FLAGS.image_folder, filenames[i] + '.' + FLAGS.image_format)
image_data = tf.gfile.FastGFile(image_filename, 'rb').read()
height, width = image_reader.read_image_dims(image_data)
seg_data = image_data
# Convert to tf example.
example = build_data.image_seg_to_tfexample(
image_data, filenames[i], height, width, seg_data)
tfrecord_writer.write(example.SerializeToString())
sys.stdout.write('\n')
sys.stdout.flush()
def _convert_dataset(dataset_split, dataset_dir, dataset_label_dir):
image_reader = build_data.ImageReader('jpeg', channels=3)
image_names = os.listdir(FLAGS.train_image_folder)
image_names[:] = [x for x in image_names if not x.startswith('.')]
image_names.sort()
output_filename = '%s_%s.tfrecord' % ('Carvana', dataset_split)
output_filename = os.path.join(FLAGS.build_datadir, output_filename)
with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer:
total = 0
for idx, image_name in enumerate(image_names):
if idx % 5 == 0:
print('total', total, 'file(s), process', idx, 'file(s).')
data_path = os.path.join(dataset_dir, image_name)
label_path = os.path.join(dataset_label_dir,
image_name[:-4] + '_mask.png')
image_data = tf.gfile.GFile(data_path, 'rb').read()
seg_data = tf.gfile.GFile(label_path, 'rb').read()
height, width = image_reader.read_image_dims(image_data)
try:
tf_example = build_data.image_seg_to_tfexample(
image_data, image_name, height, width, seg_data)
if tf_example is not None:
tfrecord_writer.write(tf_example.SerializeToString())
except ValueError:
tf.logging.warning('Invalid example:', image_name,
', ignorig.')
total += 1
print('total', total, 'file(s), process', idx, 'file(s).')
def _convert_siamese_dataset(split,
image_folder,
annos,
tfrecord_base_dir,
num_shards=10):
# shuffle annos
sf_annos = copy.deepcopy(annos)
random.shuffle(sf_annos)
random.shuffle(sf_annos)
random.shuffle(sf_annos)
# convert each row in annos into tfrecord
num_annos = len(sf_annos)
num_per_shard = int(math.ceil(num_annos / float(num_shards)))
image_reader = build_data.ImageReader('jpeg', channels=3)
for shard_id in range(num_shards):
output_filename = os.path.join(
tfrecord_base_dir,
'%s-%05d-of-%05d.tfrecord' % (split, shard_id, num_shards))
start_idx = shard_id * num_per_shard
end_idx = min((shard_id + 1) * num_per_shard, num_annos)
with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer:
for i in range(start_idx, end_idx):
sys.stdout.write('\r>> Converting image %d/%d shard %d' %
(i + 1, num_annos, shard_id))
sys.stdout.flush()
# Read the image and extract more information
id_a = sf_annos[i][0]
id_b = sf_annos[i][1]
im_a_name = sf_annos[i][2]
im_a = tf.gfile.FastGFile(
os.path.join(image_folder, im_a_name), 'rb').read()
im_a_h, im_a_w = image_reader.read_image_dims(im_a)
im_b_name = sf_annos[i][3]
im_b = tf.gfile.FastGFile(
os.path.join(image_folder, im_b_name), 'rb').read()
im_b_h, im_b_w = image_reader.read_image_dims(im_b)
label = int(sf_annos[i][4])
# make one tf example
example = tf.train.Example(features=tf.train.Features(
feature={
'id_a': _bytes_list_feature(id_a),
'im_a': _bytes_list_feature(im_a),
'im_a_name': _bytes_list_feature(im_a_name),
'im_a_h': _int64_list_feature(im_a_h),
'im_a_w': _int64_list_feature(im_a_w),
'id_b': _bytes_list_feature(id_b),
'im_b': _bytes_list_feature(im_b),
'im_b_name': _bytes_list_feature(im_b_name),
'im_b_h': _int64_list_feature(im_b_h),
'im_b_w': _int64_list_feature(im_b_w),
'label': _int64_list_feature(label),
}))
tfrecord_writer.write(example.SerializeToString())
sys.stdout.write('\n')
sys.stdout.flush()
def _convert_dataset(dataset_split):
"""Converts the specified dataset split to numpy format.
Args:
dataset_split: The dataset split (e.g., train, test).
Raises:
RuntimeError: If loaded image and label have different shape.
"""
dataset = os.path.basename(dataset_split)[:-4]
sys.stdout.write('Processing ' + dataset)
filenames = [x.strip('\n') for x in open(dataset_split, 'r')]
num_images = len(filenames)
image_reader = build_data.ImageReader('jpeg', channels=3)
label_reader = build_data.ImageReader('png', channels=1)
img_np = np.empty((num_images, 160, 160, 3), dtype=np.uint8)
label_np = np.empty((num_images, 160, 160), dtype=np.uint8)
for i in tqdm(range(num_images)):
# Read the image.
image_filename = os.path.join(FLAGS.image_folder,
filenames[i] + '.' + FLAGS.image_format)
# image_data = tf.gfile.FastGFile(image_filename, 'rb').read()
# height, width = image_reader.read_image_dims(image_data)
image_data = imresize(imread(image_filename), (160, 160), 'bilinear')
height, width = image_data.shape[0:2]
# Read the semantic segmentation annotation.
seg_filename = os.path.join(FLAGS.semantic_segmentation_folder,
filenames[i] + '.' + FLAGS.label_format)
# seg_data = tf.gfile.FastGFile(seg_filename, 'rb').read()
# seg_height, seg_width = label_reader.read_image_dims(seg_data)
seg_data = imresize(imread(seg_filename), (160, 160), 'nearest')
seg_height, seg_width = seg_data.shape[0:2]
if height != seg_height or width != seg_width:
raise RuntimeError('Shape mismatched between image and label.')
# print(image_data.shape)
# print(image_data.dtype)
# print(seg_data.shape)
# print(seg_data.dtype)
# exit(0)
img_np[i] = image_data.copy()
label_np[i] = seg_data.copy()
np.save(FLAGS.output_dir + 'x_' + dataset,
np.transpose(img_np, (0, 3, 1, 2)))
np.save(FLAGS.output_dir + 'y_' + dataset, label_np)
def _convert_dataset(dataset_split):
"""
convert images and annots to tfrecord format.
Args:
dataset_split: "train" or "eval"
"""
image_files = _get_files("image", dataset_split)
txt_files = _get_files("txt", dataset_split)
idx_list = list(range(len(image_files)))
assert len(image_files) == len(
txt_files
), "number of image files and number of txts files must be same, if not, check your dataset dir"
random.shuffle(idx_list)
num_images = len(image_files)
num_per_shard = int(math.ceil(num_images / float(_NUM_SHARDS)))
image_reader = build_data.ImageReader("jpg", channels=3)
if not os.path.exists(tfrecord_output_dir):
os.mkdir(tfrecord_output_dir)
for shard_id in range(_NUM_SHARDS):
shard_filename = "%s-%02d-of-%02d.tfrecord" % (dataset_split, shard_id,
_NUM_SHARDS)
output_filename = os.path.join(tfrecord_output_dir, shard_filename)
with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer:
start_idx = shard_id * num_per_shard
end_idx = min((shard_id + 1) * num_per_shard, num_images)
for i in range(start_idx, end_idx):
sys.stdout.write('\r>> Converting image %d/%d shard %d' %
(i + 1, num_images, shard_id))
sys.stdout.flush()
image_data = tf.gfile.GFile(image_files[idx_list[i]],
'rb').read()
height, width = image_reader.read_image_dims(image_data)
with open(txt_files[idx_list[i]], 'r') as txt_f:
content = txt_f.readline().strip().split(',')
points = [int(_) for _ in content]
filename = os.path.basename(image_files[idx_list[i]])
txtname = os.path.basename(txt_files[idx_list[i]])
if filename.split('.')[0] != txtname.split('.')[0]:
raise ValueError('filename != txtname')
example = build_data.image_label_to_tfexample(
image_data, filename, height, width, points)
tfrecord_writer.write(example.SerializeToString())
sys.stdout.write('\n')
sys.stdout.flush()
def _convert_dataset(dataset_split):
"""Converts the specified dataset split to TFRecord format.
Args:
dataset_split: The dataset split (e.g., train, val).
Raises:
RuntimeError: If loaded image and label have different shape, or if the
image file with specified postfix could not be found.
"""
image_files = _get_files('image', dataset_split)
label_files = _get_files('label', dataset_split)
num_images = len(image_files)
num_per_shard = int(math.ceil(num_images / float(_NUM_SHARDS)))
print 'num_images, num_labels, num_per_shard: ', num_images, len(label_files), num_per_shard
image_reader = build_data.ImageReader('jpg', channels=3)
# label_reader = build_data.ImageReader('png', channels=1)
for shard_id in range(_NUM_SHARDS):
shard_filename = '%s-%05d-of-%05d.tfrecord' % (
dataset_split, shard_id, _NUM_SHARDS)
output_filename = os.path.join(FLAGS.output_dir, shard_filename)
options = tf.python_io.TFRecordOptions(tf.python_io.TFRecordCompressionType.GZIP)
with tf.python_io.TFRecordWriter(output_filename, options=options) as tfrecord_writer:
start_idx = shard_id * num_per_shard
end_idx = min((shard_id + 1) * num_per_shard, num_images)
for i in range(start_idx, end_idx):
sys.stdout.write('\r>> Converting image %d/%d shard %d' % (
i + 1, num_images, shard_id))
sys.stdout.flush()
# Read the image.
image_data = tf.gfile.FastGFile(image_files[i], 'rb').read()
height, width = image_reader.read_image_dims(image_data)
# Read the semantic segmentation annotation.
# posemap_data = np.load(label_files[i])
with h5py.File(label_files[i], 'r') as hf:
posemap_data = hf['posemap'][:]
posemap_data[posemap_data==np.inf] = 255.
print np.sum(posemap_data), np.max(posemap_data), np.min(posemap_data)
# print seg_data.shape, seg_data.dtype
# seg_height, seg_width = label_reader.read_image_dims(seg_data)
posemap_height, posemap_width = posemap_data.shape[0], posemap_data.shape[1]
if height != posemap_height or width != posemap_width:
raise RuntimeError('Shape mismatched between image and label.')
# Convert to tf example.
re_match = _IMAGE_FILENAME_RE.search(image_files[i])
if re_match is None:
raise RuntimeError('Invalid image filename: ' + image_files[i])
filename = os.path.basename(re_match.group(1))
example = build_data.image_posemap_to_tfexample(
image_data, filename, height, width, posemap_data)
tfrecord_writer.write(example.SerializeToString())
sys.stdout.write('\n')
sys.stdout.flush()
def _convert_dataset(dataset_split, image_names, labels_df):
"""Converts the ADE20k dataset into into tfrecord format.
Args:
dataset_split: Dataset split (e.g., train, val).
image_ids: ...
labels_df: ...
Raises:
RuntimeError: If loaded image and label have different shape.
"""
num_images = len(image_names)
num_per_shard = int(math.ceil(num_images / float(_NUM_SHARDS)))
image_class_id_to_rle_mask = dict(
zip(labels_df.ImageId_ClassId, labels_df.EncodedPixels))
image_reader = build_data.ImageReader('jpeg', channels=3)
# label_reader = build_data.ImageReader('png', channels=1)
for shard_id in range(_NUM_SHARDS):
output_filename = os.path.join(
FLAGS.output_dir, '%s-%05d-of-%05d.tfrecord' %
(dataset_split, shard_id, _NUM_SHARDS))
with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer:
start_idx = shard_id * num_per_shard
end_idx = min((shard_id + 1) * num_per_shard, num_images)
for i in range(start_idx, end_idx):
sys.stdout.write('\r>> Converting image %d/%d shard %d' %
(i + 1, num_images, shard_id))
sys.stdout.flush()
# Read the image.
image_name = image_names[i]
image_data = tf.gfile.FastGFile(image_name, 'rb').read()
height, width = image_reader.read_image_dims(image_data)
# Read the semantic segmentation annotation.
image_id = image_name.split('/')[-1].split('.')[0]
rle_masks = [
image_class_id_to_rle_mask['{}.jpg_{}'.format(
image_id, i + 1)] for i in range(4)
]
masks = [
image_utils.rle_to_mask(rle_mask, height, width)
for rle_mask in rle_masks
]
mask = masks_to_mask(masks)
mask_data = image_utils.numpy_to_bytes(mask, 'png')
# Convert to tf example.
example = build_data.image_seg_to_tfexample(
image_data, image_name, height, width, mask_data)
tfrecord_writer.write(example.SerializeToString())
sys.stdout.write('\n')
sys.stdout.flush()
def convert_dataset(dataset, names, num_shared=4):
sys.stdout.write('Processing ' + dataset)
sys.stdout.write('\n')
sys.stdout.flush()
num = len(names)
num_per_shard = int(math.ceil(num / float(num_shared)))
image_reader = build_data.ImageReader('jpg', channels=3)
label_reader = build_data.ImageReader('png', channels=1)
for shard_id in range(num_shared):
output_filename = os.path.join(
dataset_folder, 'tfrecord',
'%s-%05d-of-%05d.tfrecord' % (dataset, shard_id, num_shared))
with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer:
start_idx = shard_id * num_per_shard
end_idx = min((shard_id + 1) * num_per_shard, num)
for i in range(start_idx, end_idx):
sys.stdout.write('\r>> Converting image %d/%d shard %d' %
(i + 1, num, shard_id))
sys.stdout.flush()
# Read the image.
image_filename = os.path.join(dataset_folder, 'train',
names[i] + data_format)
image_data = tf.gfile.FastGFile(image_filename, 'rb').read()
height, width = image_reader.read_image_dims(image_data)
# Read the semantic segmentation annotation.
seg_filename = os.path.join(dataset_folder, 'seg',
names[i] + seg_format)
seg_data = tf.gfile.FastGFile(seg_filename, 'rb').read()
seg_height, seg_width = label_reader.read_image_dims(seg_data)
if height != seg_height or width != seg_width:
raise RuntimeError(
'Shape mismatched between image and label.')
# Convert to tf example.
example = build_data.image_seg_to_tfexample(
image_data, names[i], height, width, seg_data)
tfrecord_writer.write(example.SerializeToString())
sys.stdout.write('\n')
sys.stdout.flush()
def convert_dataset(dataset, is_ship_list, nan_ship_list, train_df):
sys.stdout.write('Processing ' + dataset + '\n')
sys.stdout.flush()
min_num = min(len(is_ship_list), len(nan_ship_list))
#min_num = 100
sys.stdout.write('Number of train samples: %d\n' % (2 * min_num))
sys.stdout.flush()
is_ship_list = random.sample(is_ship_list, min_num)
nan_ship_list = random.sample(nan_ship_list, min_num)
num_per_shard = int(math.ceil(min_num / float(_NUM_SHARDS)))
image_reader = build_data.ImageReader('jpg', channels=3)
label_reader = build_data.ImageReader('png', channels=1)
for shard_id in range(_NUM_SHARDS):
output_filename = os.path.join(
FLAGS.output_dir,
'%s-%05d-of-%05d.tfrecord' % (dataset, shard_id, _NUM_SHARDS))
with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer:
start_idx = shard_id * num_per_shard
end_idx = min((shard_id + 1) * num_per_shard, min_num)
for i in range(start_idx, end_idx):
sys.stdout.write(
'\r>> Converting pair of images %d/%d shard %d' %
(i + 1, min_num, shard_id))
sys.stdout.flush()
if dataset == 'train':
aug_write_tfexample(is_ship_list[i], image_reader,
label_reader, train_df,
tfrecord_writer)
elif dataset == 'val':
write_tfexample(is_ship_list[i], image_reader,
label_reader, train_df, tfrecord_writer)
else:
raise RuntimeError('Dataset must be val/train.')
write_tfexample(nan_ship_list[i], image_reader, label_reader,
train_df, tfrecord_writer)
sys.stdout.write('\n')
sys.stdout.flush()
def _convert_dataset(dataset_split):
"""Converts the specified dataset split to TFRecord format.
Args:
dataset_split: The dataset split (e.g., train, test).
"""
dataset = os.path.basename(dataset_split)[:-4]
sys.stdout.write('Processing ' + dataset)
filenames = [x.strip('\n') for x in open(dataset_split, 'r')]
coordinate_filenames = 'figCon'
num_images = len(filenames)
num_per_shard = int(math.ceil(num_images / float(_NUM_SHARDS)))
image_reader = build_data.ImageReader('jpg', channels=1) # 3
# Read the finger roots data.
coordinates_filename = os.path.join(
FLAGS.coordinates_filename_folder,
coordinate_filenames + '.' + FLAGS.label_format)
coordinates_data = tf.gfile.FastGFile(coordinates_filename, 'r').read()
label_reader = build_data.TxtReader(coordinates_data)
for shard_id in range(_NUM_SHARDS):
output_filename = os.path.join(
FLAGS.output_dir,
'%s-%05d-of-%05d.tfrecord' % (dataset, shard_id, _NUM_SHARDS))
with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer:
start_idx = shard_id * num_per_shard
end_idx = min((shard_id + 1) * num_per_shard, num_images)
for i in range(start_idx, end_idx):
sys.stdout.write('\r>> Converting image %d/%d shard %d' % (
i + 1, len(filenames), shard_id))
sys.stdout.flush()
# Read the image.
image_filename = os.path.join(
FLAGS.image_folder, filenames[i] + '.' + FLAGS.image_format)
image_data = tf.gfile.FastGFile(image_filename, 'rb').read()
# tf.cast(image_data, tf.float64)
# image_data = tf.math.l2_normalize(image_data)
height, width = image_reader.read_image_dims(image_data)
finger_roots = label_reader.read_coordinates_data(filenames[i] + '.' + FLAGS.image_format)
if len(finger_roots) != 6:
raise ValueError("finger roots data: %s is illegal", finger_roots)
# Convert to tf example.
example = build_data.image_coordinates_to_tfexample(
image_data, filenames[i], height, width, finger_roots)
tfrecord_writer.write(example.SerializeToString())
sys.stdout.write('\n')
sys.stdout.flush()
def _convert_dataset(seq_idx):
"""Converts the specified dataset split to TFRecord format.
Args:
dataset_split: The dataset split (e.g., train, val).
Raises:
RuntimeError: If loaded image and label have different shape, or if the
image file with specified postfix could not be found.
"""
image_files = _get_files('image', seq_idx)
label_files = _get_files('label', seq_idx)
image_reader = build_data.ImageReader('png', channels=3)
label_reader = build_data.ImageReader('png', channels=1)
shard_filename = 'seq-%05d-of-%05d.tfrecord' % (0, 1)
output_filename = os.path.join(usr_dir, FLAGS.cityscapes_root,
FLAGS.output_dir, shard_filename)
with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer:
for i in xrange(len(image_files)):
# Read the image.
image_data = tf.gfile.FastGFile(image_files[i], 'rb').read()
height, width = image_reader.read_image_dims(image_data)
# Read the semantic segmentation annotation.
seg_data = tf.gfile.FastGFile(label_files[i], 'rb').read()
seg_height, seg_width = label_reader.read_image_dims(seg_data)
if height != seg_height or width != seg_width:
raise RuntimeError('Shape mismatched between image and label.')
# Convert to tf example.
re_match = _IMAGE_FILENAME_RE.search(image_files[i])
if re_match is None:
raise RuntimeError('Invalid image filename: ' + image_files[i])
filename = os.path.basename(re_match.group(1))
example = build_data.image_seg_to_tfexample(
image_data, filename, height, width, seg_data)
tfrecord_writer.write(example.SerializeToString())
def WriteRecords(args, datasets, imbddata):
'''
datasets = [{'name':'training', 'ratio':0.7}, {'name':'validation', 'ratio':0.3}]
imbddata = []
'''
# shuffle records between datasets and shards
random.shuffle(imbddata)
image_reader = build_data.ImageReader('jpeg', channels=3)
start = 0
numEntries = len(imbddata)
for ids, dataset in enumerate(datasets):
for shard_id in range(args.shards):
output_filename = os.path.join(
args.out, '%s-%05d-of-%05d.tfrecord' %
(dataset['name'], shard_id, args.shards))
if (ids == len(datasets) - 1 and shard_id == args.shards - 1):
stop = numEntries
else:
groupSize = int(numEntries * dataset['ratio'] / args.shards)
stop = start + groupSize
print('{} start {} stop {}'.format(output_filename, start, stop))
pb = ProgressBar(total=stop - start,
prefix='{}'.format(dataset['name']),
suffix='of {}'.format(stop - start),
decimals=3,
length=75,
fill='%',
zfill='-')
with tf.io.TFRecordWriter(output_filename) as tfrecord_writer:
for i in range(start, stop):
example = Example(args, i, imbddata[i], image_reader)
tfrecord_writer.write(example.SerializeToString())
if i % 100 == 0:
pb.print_progress_bar(i - start)
pb.print_progress_bar(stop - start)
start = stop
sys.stdout.write('\n')
sys.stdout.flush()
def _convert_dataset():
"""Converts the specified dataset split to TFRecord format.
Args:
dataset_split: The dataset split (e.g., train, val).
Raises:
RuntimeError: If loaded image and label have different shape, or if the
image file with specified postfix could not be found.
"""
image_files = _get_files('image')
print(len(image_files))
num_images = len(image_files)
num_per_shard = int(math.ceil(num_images / float(_NUM_SHARDS)))
image_reader = build_data.ImageReader('png', channels=3)
for shard_id in range(_NUM_SHARDS):
shard_filename = '%s-%05d-of-%05d.tfrecord' % ('train', shard_id,
_NUM_SHARDS)
output_filename = os.path.join(FLAGS.output_dir, shard_filename)
with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer:
start_idx = shard_id * num_per_shard
end_idx = min((shard_id + 1) * num_per_shard, num_images)
for i in range(start_idx, end_idx):
sys.stdout.write('\r>> Converting image %d/%d shard %d' %
(i + 1, num_images, shard_id))
sys.stdout.flush()
# Read the image.
image_data = tf.gfile.FastGFile(image_files[i], 'rb').read()
height, width = image_reader.read_image_dims(image_data)
# Convert to tf example.
re_match = _IMAGE_FILENAME_RE.search(image_files[i])
if re_match is None:
raise RuntimeError('Invalid image filename: ' +
image_files[i])
filename = os.path.basename(re_match.group(1))
example = build_data.image_to_tfexample(
image_data, filename, height, width)
tfrecord_writer.write(example.SerializeToString())
sys.stdout.write('\n')
sys.stdout.flush()
def _convert_dataset(dataset, train_isship_list, train_nanship_list, train_df):
sys.stdout.write('Processing ' + dataset)
min_num = min(len(train_isship_list), len(train_nanship_list))
#min_num = 100
print('Number of train samples: ', 2 * min_num)
train_isship_list = random.sample(train_isship_list, min_num)
train_nanship_list = random.sample(train_nanship_list, min_num)
num_per_shard = int(math.ceil(min_num / float(_NUM_SHARDS)))
image_reader = build_data.ImageReader('jpg', channels=3)
for shard_id in range(_NUM_SHARDS):
output_filename = os.path.join(
FLAGS.output_dir,
'%s-%05d-of-%05d.tfrecord' % (dataset, shard_id, _NUM_SHARDS))
with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer:
start_idx = shard_id * num_per_shard
end_idx = min((shard_id + 1) * num_per_shard, min_num)
for i in range(start_idx, end_idx):
sys.stdout.write(
'\r>> Converting pair of images %d/%d shard %d' %
(i + 1, min_num, shard_id))
sys.stdout.flush()
write_tfexample(train_isship_list[i],
image_reader,
train_df,
tfrecord_writer,
label=1)
write_tfexample(train_nanship_list[i],
image_reader,
train_df,
tfrecord_writer,
label=0)
sys.stdout.write('\n')
sys.stdout.flush()
ITL_anoDir = 'e:/temp/itl/models/research/deeplab/datasets/itl/segmentationclass/'
ITL_imageDir = 'e:/temp/itl/models/research/deeplab/datasets/itl/jpegimages/'
ITL_listDir = 'e:/temp/itl/models/research/deeplab/datasets/itl/imagesets/'
ITL_outputDir = 'e:/temp/itl/models/research/deeplab/datasets/itl/tfrecord/'
ITL_numShards = 4
ITL_datasetSplits = tf.gfile.Glob(os.path.join(ITL_listDir, '*.txt'))
for ITL_datasetSplit in ITL_datasetSplits:
ITL_dataset = os.path.basename(ITL_datasetSplit)[:-4]
print('Processing ' + ITL_dataset)
ITL_filenames = [x.strip('\n') for x in open(ITL_datasetSplit, 'r')]
ITL_numImages = len(ITL_filenames)
ITL_numPerShard = int(math.ceil(ITL_numImages / float(ITL_numShards)))
ITL_imageReader = build_data.ImageReader('jpeg', channels=3)
ITL_labelReader = build_data.ImageReader('png', channels=1)
for ITL_shardId in range(ITL_numShards):
ITL_outputFilename = os.path.join(ITL_outputDir, '%s-%05d-of-%05d.tfrecord' % (ITL_dataset, ITL_shardId, ITL_numShards))
with tf.python_io.TFRecordWriter(ITL_outputFilename) as tfrecord_writer:
ITL_startIdx = ITL_shardId * ITL_numPerShard
ITL_endIdx = min((ITL_shardId + 1) * ITL_numPerShard, ITL_numImages)
for i in range(ITL_startIdx, ITL_endIdx):
print('\r>> Converting image %d/%d shard %d' % (i + 1, len(ITL_filenames), ITL_shardId))
sys.stdout.flush()
ITL_imageFilename = os.path.join(ITL_imageDir, ITL_filenames[i] + '.jpg')
ITL_imageData = tf.gfile.FastGFile(ITL_imageFilename, 'rb').read()
height, width = ITL_imageReader.read_image_dims(ITL_imageData)
ITL_anoFilename = os.path.join(ITL_anoDir, ITL_filenames[i] + '.png')
ITL_anoData = tf.gfile.FastGFile(ITL_anoFilename, 'rb').read()
def _convert_dataset(dataset_split):
dataset = os.path.basename(dataset_split)[:-4] # train or test
print('Processing ' + dataset)
filenames = [x.strip('\n') for x in open(dataset_split, 'r')]
num_images = len(filenames)
num_per_shard = int(math.ceil(num_images / float(_NUM_SHARDS)))
image_reader = build_data.ImageReader('jpeg', channels=3)
label_reader = build_data.ImageReader('png', channels=1)
superpixel_reader = build_data.ImageReader('png', channels=1)
for shard_id in range(_NUM_SHARDS):
output_filename = os.path.join(
FLAGS.output_dir,
'%s-%05d-of-%05d.tfrecord' % (dataset, shard_id, _NUM_SHARDS))
with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer:
start_idx = shard_id * num_per_shard
end_idx = min((shard_id + 1) * num_per_shard, num_images)
for i in range(start_idx, end_idx):
print('\r>> Converting image %d/%d shard %d' %
(i + 1, len(filenames), shard_id))
if dataset == 'train':
image_filename = os.path.join(
FLAGS.train_image_folder,
filenames[i] + '.' + FLAGS.image_format)
seg_filename = os.path.join(
FLAGS.train_semantic_segmentation_folder,
filenames[i] + '.' + FLAGS.label_format)
superpixel_filename = os.path.join(
FLAGS.train_superpixel_folder,
filenames[i] + '.' + FLAGS.label_format)
elif dataset == 'test':
image_filename = os.path.join(
FLAGS.test_image_folder,
filenames[i] + '.' + FLAGS.image_format)
seg_filename = os.path.join(
FLAGS.test_semantic_segmentation_folder,
filenames[i] + '.' + FLAGS.label_format)
superpixel_filename = os.path.join(
FLAGS.test_superpixel_folder,
filenames[i] + '.' + FLAGS.label_format)
else:
assert False
# Read the image.
image_data = tf.gfile.FastGFile(image_filename, 'rb').read()
height, width = image_reader.read_image_dims(image_data)
# Read the semantic segmentation annotation.
seg_data = tf.gfile.FastGFile(seg_filename, 'rb').read()
seg_height, seg_width = label_reader.read_image_dims(seg_data)
# Read the super pixels.
superpixel_data = tf.gfile.FastGFile(superpixel_filename,
'rb').read()
sp_height, sp_width = superpixel_reader.read_image_dims(
superpixel_data)
if not (height == seg_height == sp_height
and width == seg_width == sp_width):
raise RuntimeError(
'Shape mismatched between image, label and super pixel map.'
)
# Convert to tf example.
example = build_data.image_seg_sp_to_tfexample(
image_data, filenames[i], height, width, seg_data,
superpixel_data)
tfrecord_writer.write(example.SerializeToString())
print('\n')
def _convert_dataset(dataset_split, _NUM_SHARDS, structure_path, plane):
"""Converts the specified dataset split to TFRecord format.
Args:
dataset_split: The dataset split (e.g., train, test).
Raises:
RuntimeError: If loaded image and label have different shape.
"""
image_folder = os.path.join(structure_path, 'processed', 'PNGImages')
semantic_segmentation_folder = os.path.join(structure_path, 'processed',
'SegmentationClass')
image_format = label_format = 'png'
if not os.path.exists(
os.path.join(structure_path, 'tfrecord' + '_' + plane)):
os.makedirs(os.path.join(structure_path, 'tfrecord' + '_' + plane))
dataset = os.path.basename(dataset_split)[:-4]
sys.stdout.write('Processing ' + dataset)
filenames = [x.strip('\n') for x in open(dataset_split, 'r')]
filenames.sort()
# random.shuffle(filenames)
print(filenames)
num_images = len(filenames)
num_per_shard = int(math.ceil(num_images / float(_NUM_SHARDS)))
image_reader = build_data.ImageReader('png', channels=3)
label_reader = build_data.ImageReader('png', channels=1)
for shard_id in range(_NUM_SHARDS):
output_filename = os.path.join(
structure_path, 'tfrecord' + '_' + plane,
'%s-%05d-of-%05d.tfrecord' % (dataset, shard_id, _NUM_SHARDS))
with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer:
start_idx = shard_id * num_per_shard
end_idx = min((shard_id + 1) * num_per_shard, num_images)
for i in range(start_idx, end_idx):
sys.stdout.write('\r>> Converting image %d/%d shard %d' %
(i + 1, len(filenames), shard_id))
sys.stdout.flush()
# Read the image.
image_filename = os.path.join(
image_folder, filenames[i] + '.' + image_format)
image_data = tf.gfile.FastGFile(image_filename, 'rb').read()
height, width = image_reader.read_image_dims(image_data)
# Read the semantic segmentation annotation.
seg_filename = os.path.join(semantic_segmentation_folder,
filenames[i] + '.' + label_format)
seg_data = tf.gfile.FastGFile(seg_filename, 'rb').read()
seg_height, seg_width = label_reader.read_image_dims(seg_data)
if height != seg_height or width != seg_width:
raise RuntimeError(
'Shape mismatched between image and label.')
# Convert to tf example.
example = build_data.image_seg_to_tfexample(
image_data, str.encode(filenames[i], 'utf-8'), height,
width, seg_data)
tfrecord_writer.write(example.SerializeToString())
sys.stdout.write('\n')
sys.stdout.flush()
def _convert_dataset(dataset_split,
dataset_name='',
class_ignore_value=255,
num_shards=1,
num_imgs=-1,
remaining_imgs_type=None,
remaining_imgs_num_or_ratio=None,
overwrite=False,
verbose=True,
shuffle=False,
shuffle_seed=1234):
"""Converts the specified dataset split to TFRecord format.
Args:
dataset_split: The dataset split (e.g., train, val).
dataset_name: The dataset name (e.g., train_hints, val_hints). Default is set to dataset_split. This is the name of the tfrecord.
remaining_imgs_type: if num_imgs is set, what should we do we remaining images if remaining_imgs_num_or_ratio is set?
None: Use remaining image labels as-is
path -- string: Use labels from path (use alternative labels)
remaining_imgs_num_or_ratio::
None: don't use remaining images (same as 0 or 0.0)
ratio -- float between 0 and 1: use ratio * num_remaining_imgs remaining images.
num -- integer: use num remaining images
Raises:
RuntimeError: If loaded image and label have different shape, or if the
image file with specified postfix could not be found.
"""
sys.stdout.flush()
print('###############################################')
sys.stdout.write('\rWorking on: {}\n'.format(dataset_name))
image_files = []
label_files = []
print('Using full res images and labels...')
image_files.extend(_get_files('image', dataset_split))
if 'coarse' in dataset_name:
label_files.extend(_get_files('label_coarse', dataset_split))
elif 'block' in dataset_name:
label_files.extend(_get_files('label_block', dataset_split))
else:
label_files.extend(_get_files('label', dataset_split))
if num_imgs < 0:
num_images = len(image_files)
else:
num_images = num_imgs
remaining_imgs_label_files = None
num_remaining_imgs_num = None
if remaining_imgs_num_or_ratio is not None:
if type(remaining_imgs_num_or_ratio) == float:
assert 0 <= remaining_imgs_num_or_ratio <= 1
num_remaining_images = int(remaining_imgs_num_or_ratio *
(len(image_files) - num_images))
if type(remaining_imgs_num_or_ratio) == int:
assert 0 <= remaining_imgs_num_or_ratio
num_remaining_images = min(remaining_imgs_num_or_ratio,
(len(image_files) - num_images))
if remaining_imgs_type is None:
remaining_imgs_label_files = list(label_files)
elif type(remaining_imgs_type) == str:
print("Searching {} for label files.")
remaining_imgs_label_files = []
for root, dirnames, filenames in os.walk(remaining_imgs_type):
for filename in fnmatch.filter(filenames, "*"):
remaining_imgs_label_files.append(
os.path.join(root, filename))
remaining_imgs_label_files = sorted(remaining_imgs_label_files)
assert len(remaining_imgs_label_files) == len(
label_files
), 'Expected {} alternative label files; found {}'.format(
len(label_files), len(remaining_imgs_label_files))
else:
raise TypeError("remaining_imgs_type should be a string or None")
if shuffle:
shuffled_idxs = np.arange(len(image_files))
np.random.seed(shuffle_seed)
np.random.shuffle(shuffled_idxs)
print('Using indices {} ...'.format(shuffled_idxs[:10]))
# Image, label, boundary distance map
image_files = np.array(image_files)[shuffled_idxs]
label_files = np.array(label_files)[shuffled_idxs]
# Alternative label files
if remaining_imgs_label_files is not None:
remaining_imgs_label_files = np.array(
remaining_imgs_label_files)[shuffled_idxs]
# Concat num_images label_files with num_remaining_images_label_files
label_files = list(label_files)[:num_images] + list(
remaining_imgs_label_files)[num_images:num_images +
num_remaining_images]
assert len(label_files) == num_images + num_remaining_images, (
len(label_files), num_images, num_remaining_images)
num_images = num_images + num_remaining_images
if not shuffle and remaining_imgs_label_files is not None:
raise NotImplementedError("This is not going to work; check the code")
num_per_shard = int(math.ceil(num_images / float(num_shards)))
image_reader = build_data.ImageReader('png', channels=3)
label_reader = build_data.ImageReader('png', channels=1)
for shard_id in range(num_shards):
if dataset_name == '':
dataset_name = dataset_split
shard_filename = '%s-%05d-of-%05d.tfrecord' % (dataset_name, shard_id,
num_shards)
output_filename = os.path.join(FLAGS.output_dir, shard_filename)
if os.path.exists(output_filename) and not overwrite:
print('File exists. Skipping. {}'.format(output_filename))
continue
with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer:
start_idx = shard_id * num_per_shard
end_idx = min((shard_id + 1) * num_per_shard, num_images)
for i in range(start_idx, end_idx):
sys.stdout.flush()
sys.stdout.write('\r>> Converting image %d/%d shard %d' %
(i + 1, num_images, shard_id))
#sys.stdout.flush()
# Read the image.
image_data = tf.gfile.FastGFile(image_files[i], 'rb').read()
height, width = image_reader.read_image_dims(image_data)
# Read the semantic segmentation annotation.
seg_data = tf.gfile.FastGFile(label_files[i], 'rb').read()
seg_height, seg_width = label_reader.read_image_dims(seg_data)
if verbose:
sys.stdout.write('\r\nUsing\n {}\n {}\n'.format(
image_files[i], label_files[i]))
if height != seg_height or width != seg_width:
raise RuntimeError(
'Shape mismatched between image and label.')
# Convert to tf example.
base_filename = os.path.basename(image_files[i]).\
replace(_POSTFIX_MAP['image']+'.'+_DATA_FORMAT_MAP['image'], '')
filename = base_filename
example = build_data.image_seg_to_tfexample(
image_data=image_data,
filename=filename,
height=height,
width=width,
seg_data=seg_data,
)
tfrecord_writer.write(example.SerializeToString())
sys.stdout.write('\n')
sys.stdout.flush()
def _convert_dataset(dataset_split):
"""Converts the specified dataset split to TFRecord format.
Args:
dataset_split: The dataset split (e.g., train, test).
Raises:
RuntimeError: If loaded image and label have different shape.
"""
# get the image and annotation path from dataset text.
dataset = os.path.basename(dataset_split)[:-4]
sys.stdout.write('Processing ' + dataset + '\n')
img_list = []
ann_list = []
img_dir = FLAGS.image_dir
ann_dir = FLAGS.annotation_dir
with open(dataset_split, 'r') as f:
for row in f:
row = row.strip()
img_path = os.path.join(img_dir, row) + '.jpg'
ann_path = os.path.join(ann_dir, row) + '.png'
img_list.append(img_path)
ann_list.append(ann_path)
image_reader = build_data.ImageReader(channels=3, img_format='jpeg')
label_reader = build_data.ImageReader(channels=1)
if not os.path.exists(FLAGS.output_dir):
os.makedirs(FLAGS.output_dir)
assert len(img_list) == len(ann_list)
# create each video type tfrecord
num_img = len(img_list)
num_img_per_shard = math.ceil(num_img / NUM_SHARD)
print('The number of %s image: %d' % (dataset_split, num_img))
for shard_id in range(NUM_SHARD):
output_filename = os.path.join(
FLAGS.output_dir,
'%s-%05d-of-%05d.tfrecord' % (dataset, shard_id + 1, NUM_SHARD))
start_idx = shard_id * num_img_per_shard
end_idx = min((shard_id + 1) * num_img_per_shard, num_img)
with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer:
for i in range(start_idx, end_idx):
img_path = img_list[i]
ann_path = ann_list[i]
sys.stdout.write('\r>> Converting image %d/%d shard %d' %
(i + 1, num_img, shard_id))
sys.stdout.flush()
# Read the image
img_data = tf.gfile.GFile(img_path, 'rb').read()
height, width = image_reader.read_image_dims(img_data)
ann_data = tf.gfile.GFile(ann_path, 'rb').read()
ann_height, ann_width = label_reader.read_image_dims(ann_data)
if height != ann_height or width != ann_width:
raise RuntimeError(
'Shape mismatched between image and annotations.')
# Convert to tf example.
example = build_data.image_seg_to_tfexample(
os.path.basename(img_path), img_data, height, width,
ann_data)
tfrecord_writer.write(example.SerializeToString())
sys.stdout.write('\n')
sys.stdout.flush()
def _convert_dataset(db_name):
"""Converts the specified dataset split to TFRecord format.
Args:
db_name: The dataset split (e.g., train, test).
Raises:
RuntimeError: If loaded image and label have different shape.
"""
output_dir = os.path.join(FLAGS.db_root_dir, FLAGS.output_dir, 'tfrecord')
sys.stdout.write('Processing {}\n\n'.format(db_name))
images = os.path.join(FLAGS.db_root_dir, db_name, 'images',
'*.{}'.format(FLAGS.image_format))
print('Reading images from: {}'.format(images))
image_filenames = glob.glob(images)
if image_filenames is None:
raise SystemError('No images found at {}'.format(images))
if FLAGS.create_dummy_labels:
labels_path = os.path.join(FLAGS.db_root_dir, db_name, 'labels')
if not os.path.isdir(labels_path):
os.makedirs(labels_path)
print('Creating dummy labels at: {}'.format(labels_path))
for image_filename in image_filenames:
image = misc.imread(image_filename)
height, width, _ = image.shape
dummy_label = np.zeros((height, width), dtype=np.uint8)
out_fname = os.path.splitext(
os.path.basename(image_filename))[0] + '.{}'.format(
FLAGS.label_format)
misc.imsave(os.path.join(labels_path, out_fname), dummy_label)
print('Done')
labels = os.path.join(FLAGS.db_root_dir, db_name, 'labels',
'*.{}'.format(FLAGS.label_format))
print('Reading labels from: {}'.format(labels))
seg_filenames = glob.glob(labels)
if seg_filenames is None:
raise SystemError('No labels found at {}'.format(labels))
# filenames = [x.strip('\n') for x in open(dataset_split, 'r')]
num_images = len(image_filenames)
num_labels = len(seg_filenames)
if num_images != num_labels:
raise SystemError(
'Mismatch between image and label file counts: {}, {}'.format(
num_images, num_labels))
num_per_shard = int(math.ceil(num_images / float(_NUM_SHARDS)))
image_reader = build_data.ImageReader('png', channels=3)
label_reader = build_data.ImageReader('png', channels=3)
if not os.path.isdir(output_dir):
os.makedirs(output_dir)
print('Writing tfrecords to: {}'.format(output_dir))
for shard_id in range(_NUM_SHARDS):
output_filename = os.path.join(
output_dir,
'%s-%05d-of-%05d.tfrecord' % (db_name, shard_id, _NUM_SHARDS))
with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer:
start_idx = shard_id * num_per_shard
end_idx = min((shard_id + 1) * num_per_shard, num_images)
for i in range(start_idx, end_idx):
sys.stdout.write('\r>> Converting image %d/%d shard %d' %
(i + 1, num_images, shard_id))
sys.stdout.flush()
image_filename = image_filenames[i]
f1 = os.path.basename(image_filename)[:-4]
seg_filename = seg_filenames[i]
f2 = os.path.basename(image_filename)[:-4]
if f1 != f2:
raise SystemError(
'Mismatch between image and label filenames: {}, {}'.
format(f1, f2))
# Read the image.
image_data = tf.gfile.FastGFile(image_filename, 'r').read()
height, width = image_reader.read_image_dims(image_data)
# Read the semantic segmentation annotation.
seg_data = tf.gfile.FastGFile(seg_filename, 'r').read()
seg_height, seg_width = label_reader.read_image_dims(seg_data)
if height != seg_height or width != seg_width:
raise RuntimeError(
'Shape mismatched between image and label.')
# Convert to tf example.
example = build_data.image_seg_to_tfexample(
image_data, image_filename, height, width, seg_data)
tfrecord_writer.write(example.SerializeToString())
sys.stdout.write('\n')
sys.stdout.flush()
def _convert_dataset(split, image_folder, anno_file, cls_name_to_index,
tfrecord_base_dir, num_shards):
split_statistics = {}
# get raw information accroding to anno_file, and shuffle it
anno_raw = []
if anno_file is not None:
with open(anno_file, 'r') as f:
for l in f.readlines():
if "Image" in l:
continue
anno_temp = l.strip().split(',')
im_name = anno_temp[0].strip()
im_name = os.path.join(image_folder,
im_name) # to absolute addrs
im_cls = anno_temp[1].strip()
im_idx = cls_name_to_index[im_cls]
anno_raw.append([im_name, im_cls, im_idx])
random.shuffle(anno_raw)
random.shuffle(anno_raw)
random.shuffle(anno_raw)
else: # test set
image_file_list = os.listdir(image_folder)
for one_im in image_file_list:
if one_im[-4:] == '.jpg':
im_name = os.path.join(image_folder, one_im)
im_cls = 'test_image'
im_idx = 999999
anno_raw.append([im_name, im_cls, im_idx])
# extract all information of one image and write it tfrecords
num_images = len(anno_raw)
num_per_shard = int(math.ceil(num_images / float(num_shards)))
image_reader = build_data.ImageReader('jpeg', channels=3)
for shard_id in range(num_shards):
output_filename = os.path.join(
tfrecord_base_dir,
'%s-%05d-of-%05d.tfrecord' % (split, shard_id, num_shards))
start_idx = shard_id * num_per_shard
end_idx = min((shard_id + 1) * num_per_shard, num_images)
with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer:
for i in range(start_idx, end_idx):
sys.stdout.write('\r>> Converting image %d/%d shard %d' %
(i + 1, num_images, shard_id))
sys.stdout.flush()
# Read the image and extract more information
image_data = tf.gfile.FastGFile(anno_raw[i][0], 'rb').read()
image_name = os.path.basename(anno_raw[i][0])
image_label = anno_raw[i][1]
image_index = anno_raw[i][2]
image_height, image_width = \
image_reader.read_image_dims(image_data)
example = tf.train.Example(features=tf.train.Features(
feature={
'image': _bytes_list_feature(image_data),
'image_name': _bytes_list_feature(image_name),
'class_name': _bytes_list_feature(image_label),
'label': _int64_list_feature(image_index),
'height': _int64_list_feature(image_height),
'width': _int64_list_feature(image_width),
}))
tfrecord_writer.write(example.SerializeToString())
anno_raw[i].extend([image_height,
image_width]) # for statistic
sys.stdout.write('\n')
sys.stdout.flush()
# for split_statistics
split_statistics['split_name'] = split
split_statistics['total_examples'] = len(anno_raw)
height_all = []
width_all = []
split_statistics['per_class_num'] = {}
for anno in anno_raw:
one_cls = anno[1]
height_all.append(anno[3])
width_all.append(anno[4])
if one_cls not in split_statistics['per_class_num']:
split_statistics['per_class_num'][one_cls] = 1
else:
split_statistics['per_class_num'][one_cls] += 1
split_statistics['height_mean'] = float(np.mean(height_all))
split_statistics['height_std'] = float(np.std(height_all))
split_statistics['width_mean'] = float(np.mean(width_all))
split_statistics['width_std'] = float(np.std(width_all))
return split_statistics
def _convert_dataset(dataset_split, dataset):
"""Converts the specified dataset split to TFRecord format.
Raises:
RuntimeError: If loaded image and label have different shape, or if the
image file with specified postfix could not be found.
"""
img_dir_name = FLAGS.img_dir_name
labels_dir_name = FLAGS.trainIDs_dir_name
output_dir = FLAGS.output_dir
image_files = dataset[0]
label_files = dataset[1]
if not (len(image_files) == len(label_files)):
raise RuntimeError('Length mismatch image and label.')
num_images = len(image_files)
num_per_shard = int(math.ceil(num_images / float(_NUM_SHARDS)))
image_reader = build_data.ImageReader('png', channels=3)
label_reader = build_data.ImageReader('png', channels=1)
for shard_id in range(_NUM_SHARDS):
shard_filename = '%s-%05d-of-%05d.tfrecord' % (dataset_split, shard_id,
_NUM_SHARDS)
output_filename = os.path.join(output_dir, shard_filename)
with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer:
start_idx = shard_id * num_per_shard
end_idx = min((shard_id + 1) * num_per_shard, num_images)
for i in range(start_idx, end_idx):
sys.stdout.write('\r>> Converting image %d/%d shard %d' %
(i + 1, num_images, shard_id))
sys.stdout.flush()
# Read the image.
image_data = tf.gfile.FastGFile(img_dir_name + image_files[i],
'rb').read()
height, width = image_reader.read_image_dims(image_data)
# Read the semantic segmentation annotation.
seg_data = tf.gfile.FastGFile(labels_dir_name + label_files[i],
'rb').read()
seg_height, seg_width = label_reader.read_image_dims(seg_data)
if height != seg_height or width != seg_width:
print(
"Shape mismatched between image and label. height. Ignoring."
)
continue
raise RuntimeError(
'Shape mismatched between image and label. height : ',
height, ' seg_height: ', seg_height, ' width: ', width,
' seg_width: ', seg_width, ' \nlabel_files[i]: ',
label_files[i], ' image_files[i]: ', image_files[i])
# Convert to tf example.
if not (image_files[i] == label_files[i]):
raise RuntimeError('image file name : ' + image_files[i] +
' is not equal to label file name : ' +
label_files[i])
filename = os.path.basename(image_files[i])
example = build_data.image_seg_to_tfexample(
image_data, filename, height, width, seg_data)
tfrecord_writer.write(example.SerializeToString())
sys.stdout.write('\n')
sys.stdout.flush()
def _convert_dataset(dataset_split):
"""Converts the specified dataset split to TFRecord format.
Args:
dataset_split: The dataset split (e.g., train, val, test).
Raises:
RuntimeError: If loaded image and label have different shape.
"""
if not os.path.isdir(FLAGS.output_dir):
os.makedirs(FLAGS.output_dir)
dataset = os.path.basename(dataset_split)[:-4]
sys.stdout.write('Processing ' + dataset)
filenames = [x.strip('\n').split(',') for x in open(dataset_split, 'r')]
num_images = len(filenames)
num_per_shard = int(math.ceil(num_images / float(_NUM_SHARDS)))
image_reader = build_data.ImageReader('png', channels=3)
label_reader = build_data.ImageReader('png', channels=1)
# print("n images:", num_images)
# print("n per shard:", num_per_shard)
# print(filenames[0])
for shard_id in range(_NUM_SHARDS):
output_filename = os.path.join(
FLAGS.output_dir,
'%s-%05d-of-%05d.tfrecord' % (dataset, shard_id, _NUM_SHARDS))
if os.path.isfile(output_filename):
continue
with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer:
start_idx = shard_id * num_per_shard
end_idx = min((shard_id + 1) * num_per_shard, num_images)
for i in range(start_idx, end_idx):
sys.stdout.write('\r>> Converting image %d/%d shard %d' %
(i + 1, len(filenames), shard_id))
sys.stdout.flush()
# Read the image data
image_filename = os.path.join(FLAGS.image_folder,
filenames[i][0])
with tf.gfile.FastGFile(image_filename, 'rb') as f:
image_data = f.read()
#image_data = tf.gfile.FastGFile(image_filename, 'rb').read()
## Resize image
image = image_reader.decode_image(image_data)
image = resize_image(image)
image = tf.image.resize_images(
image,
size=[400, 624],
method=tf.image.ResizeMethod.BILINEAR,
align_corners=False)
with tf.Session().as_default():
img = image.eval()
cv2.imwrite(
"check_localization/test_images/tfrecord-test-image-{}.png"
.format(i), img)
height, width = image.shape[:2]
image_data = image_reader.encode_image(image)
# height, width = image_reader.read_image_dims(image_data)
# Read the semantic segmentation annotation.
seg_filename = os.path.join(FLAGS.semantic_segmentation_folder,
filenames[i][1])
with tf.gfile.FastGFile(seg_filename, 'rb') as f:
seg_data = f.read()
#seg_data = tf.gfile.FastGFile(seg_filename, 'rb').read()
## Resize mask
seg_image = label_reader.decode_image(seg_data)
seg_image = resize_image(seg_image)
seg_image = tf.image.resize_images(
seg_image,
size=[400, 624],
method=tf.image.ResizeMethod.BILINEAR,
align_corners=False)
with tf.Session().as_default():
mask = seg_image.eval()
cv2.imwrite(
"check_localization/test_images/tfrecord-test-annot-{}.png"
.format(i), mask)
seg_height, seg_width = seg_image.shape[:2]
seg_data = label_reader.encode_image(seg_image)
# seg_height, seg_width = label_reader.read_image_dims(seg_data, resize=True)
if height != seg_height or width != seg_width:
raise RuntimeError(
'Shape mismatched between image and label.')
# Convert to tf example.
example = build_data.image_seg_to_tfexample(
image_data, filenames[i][0], height, width, seg_data)
tfrecord_writer.write(example.SerializeToString())
sys.stdout.write('\n')
sys.stdout.flush()
del example, tfrecord_writer,
break
def _convert_dataset(dataset_split):
"""Converts the specified dataset split to TFRecord format.
Args:
dataset_split: The dataset split (e.g., train, val, trainval)
Raises:
RuntimeError: If loaded image and label have different shape.
"""
# get the annotation paths.
pattern = os.path.join(FLAGS.annotation_dir, '%s2017' % dataset_split,
'*.png')
ann_list = sorted(glob.glob(pattern))
image_reader = build_data.ImageReader(channels=3)
label_reader = build_data.ImageReader(channels=1)
if not os.path.exists(FLAGS.output_dir):
os.makedirs(FLAGS.output_dir)
num_img = len(ann_list)
num_img_per_shard = math.ceil(num_img / NUM_SHARD)
print('The number of %s image: %d' % (dataset_split, num_img))
for shard_id in range(NUM_SHARD):
output_filename = os.path.join(
FLAGS.output_dir, '%s-%05d-of-%05d.tfrecord' %
(dataset_split, shard_id + 1, NUM_SHARD))
start_idx = shard_id * num_img_per_shard
end_idx = min((shard_id + 1) * num_img_per_shard, num_img)
with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer:
for i in range(start_idx, end_idx):
ann_path = ann_list[i]
img_path = os.path.join(
FLAGS.image_dir,
ann_path.split(os.sep)[-2],
os.path.basename(ann_path).replace('.png', '.jpg'))
sys.stdout.write('\r>> Converting image %d/%d shard %d' %
(i + 1, num_img, shard_id))
sys.stdout.flush()
if not os.path.exists(img_path):
raise ValueError('image {} dont exists'.format(img_path))
# Read the image
img_data = tf.gfile.GFile(img_path, 'rb').read()
height, width = image_reader.read_image_dims(img_data)
ann_data = tf.gfile.GFile(ann_path, 'rb').read()
ann_height, ann_width = label_reader.read_image_dims(ann_data)
if height != ann_height or width != ann_width:
raise RuntimeError(
'Shape mismatched between image and annotation.')
# Convert to tf example.
example = build_data.image_seg_to_tfexample(
os.path.basename(img_path), img_data, height, width,
ann_data)
tfrecord_writer.write(example.SerializeToString())
sys.stdout.write('\n')
sys.stdout.flush()
def _convert_dataset(dataset_split):
"""Converts the specified dataset split to TFRecord format.
Args:
dataset_split: The dataset split (e.g., train, val).
Raises:
RuntimeError: If loaded image and label have different shape.
"""
sys.stdout.write('Processing ' + dataset_split)
if dataset_split == 'train':
seg_base_dir = 'PubLayNet/SegmentationClass/train/'
raw_base_dir = 'PubLayNet/RawImages/train_data/'
else:
seg_base_dir = 'PubLayNet/SegmentationClass/val/'
raw_base_dir = 'PubLayNet/RawImages/val_data/val/'
seg_file_names = [
f for f in glob.glob(seg_base_dir + "**/*.png", recursive=True)
]
raw_file_names = [
f for f in glob.glob(raw_base_dir + "**/*.jpg", recursive=True)
]
raw_name_only = [f.split('/')[-1].split('.')[-2] for f in raw_file_names]
num_images = len(seg_file_names)
num_shards = int(math.ceil(num_images / float(num_docs_per_shard)))
image_reader = build_data.ImageReader('jpeg', channels=3)
label_reader = build_data.ImageReader('png', channels=1)
for shard_id in range(num_shards):
output_filename = os.path.join(
'./PubLayNet/tfrecord',
'%s-%05d-of-%05d.tfrecord' % (dataset_split, shard_id, num_shards))
with tf.io.TFRecordWriter(output_filename) as tfrecord_writer:
start_idx = shard_id * num_docs_per_shard
end_idx = min((shard_id + 1) * num_docs_per_shard, num_images)
for i in range(start_idx, end_idx):
sys.stdout.write('\r>> Converting image %d/%d shard %d' %
(i + 1, num_images, shard_id))
sys.stdout.flush()
# Read the semantic segmentation annotation.
seg_name_only = seg_file_names[i].split('/')[-1].split('.')[-2]
seg_data = tf.io.gfile.GFile(seg_file_names[i], 'rb').read()
'''print('/n', dataset_split, 'seg', seg_file_names[i])
print(dataset_split, 'seg', seg_name_only)'''
seg_height, seg_width = label_reader.read_image_dims(seg_data)
# Read the raw image.
name_ind = raw_name_only.index(seg_name_only)
'''print(dataset_split, 'raw', raw_file_names[name_ind])
print(dataset_split, 'raw', raw_name_only[name_ind])'''
image_data = tf.io.gfile.GFile(raw_file_names[name_ind],
'rb').read()
height, width = image_reader.read_image_dims(image_data)
if height != seg_height or width != seg_width:
#raise RuntimeError('Shape mismatched between image and label.')
print(
'The raw image and segmentation mask do not have the same dimensions.'
)
print('Skipping image {}'.format(seg_name_only))
else:
# Convert to tf example.
example = build_data.image_seg_to_tfexample(
image_data, seg_name_only, height, width, seg_data)
tfrecord_writer.write(example.SerializeToString())
sys.stdout.write('\n')
sys.stdout.flush()
