def _convert_dataset(dataset_split):
"""Converts the specified dataset split to TFRecord format.
Args:
dataset_split: The dataset split (e.g., train, test).
Raises:
RuntimeError: If loaded image and label have different shape.
"""
dataset = os.path.basename(dataset_split)[:-4]
sys.stdout.write('Processing ' + dataset)
filenames = [x.strip('\n') for x in open(dataset_split, 'r')]
num_images = len(filenames)
num_per_shard = int(math.ceil(num_images / float(_NUM_SHARDS)))
image_reader = build_data.ImageReader('jpeg', channels=3)
label_reader = build_data.ImageReader('png', channels=1)
for shard_id in range(_NUM_SHARDS):
output_filename = os.path.join(
FLAGS.output_dir,
'%s-%05d-of-%05d.tfrecord' % (dataset, shard_id, _NUM_SHARDS))
with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer:
start_idx = shard_id * num_per_shard
end_idx = min((shard_id + 1) * num_per_shard, num_images)
for i in range(start_idx, end_idx):
sys.stdout.write('\r>> Converting image %d/%d shard %d' %
(i + 1, len(filenames), shard_id))
sys.stdout.flush()
# Read the image.
image_filename = os.path.join(
FLAGS.image_folder,
filenames[i] + '.' + FLAGS.image_format)
image_data = tf.gfile.FastGFile(image_filename, 'rb').read()
height, width = image_reader.read_image_dims(image_data)
# Read the semantic segmentation annotation.
if dataset_split[-8:] != 'test.txt':
seg_filename = os.path.join(
FLAGS.semantic_segmentation_folder,
filenames[i] + '.' + FLAGS.label_format)
seg_data = tf.gfile.FastGFile(seg_filename, 'rb').read()
seg_height, seg_width = label_reader.read_image_dims(
seg_data)
if height != seg_height or width != seg_width:
raise RuntimeError(
'Shape mismatched between image and label.')
# Convert to tf example.
example = build_data.image_seg_to_tfexample(
image_data, filenames[i], height, width, seg_data)
tfrecord_writer.write(example.SerializeToString())
else:
seg_data = None
example = build_data.image_seg_to_tfexample(
image_data, filenames[i], height, width, seg_data)
tfrecord_writer.write(example.SerializeToString())
sys.stdout.write('\n')
sys.stdout.flush()
def _convert_dataset(dataset_split):
"""Converts the specified dataset split to TFRecord format.
Args:
dataset_split: The dataset split (e.g., train, val).
Raises:
RuntimeError: If loaded image and label have different shape, or if the
image file with specified postfix could not be found.
"""
image_files = _get_files('image', dataset_split)
label_files = _get_files('label', dataset_split)
num_images = len(image_files)
num_per_shard = int(math.ceil(num_images / float(_NUM_SHARDS)))
image_reader = build_data.ImageReader('png', channels=3)
label_reader = build_data.ImageReader('png', channels=1)
for shard_id in range(_NUM_SHARDS):
shard_filename = '%s-%05d-of-%05d.tfrecord' % (dataset_split, shard_id,
_NUM_SHARDS)
output_filename = os.path.join(FLAGS.output_dir, shard_filename)
with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer:
start_idx = shard_id * num_per_shard
end_idx = min((shard_id + 1) * num_per_shard, num_images)
for i in range(start_idx, end_idx):
sys.stdout.write('\r>> Converting image %d/%d shard %d' %
(i + 1, num_images, shard_id))
sys.stdout.flush()
# Read the image.
image_data = tf.gfile.FastGFile(image_files[i], 'rb').read()
height, width = image_reader.read_image_dims(image_data)
if dataset_split != 'test':
# Read the semantic segmentation annotation.
seg_data = tf.gfile.FastGFile(label_files[i], 'rb').read()
seg_height, seg_width = label_reader.read_image_dims(
seg_data)
if height != seg_height or width != seg_width:
raise RuntimeError(
'Shape mismatched between image and label.')
# Convert to tf example.
re_match = _IMAGE_FILENAME_RE.search(image_files[i])
if re_match is None:
raise RuntimeError('Invalid image filename: ' +
image_files[i])
filename = os.path.basename(re_match.group(1))
if dataset_split != 'test':
example = build_data.image_seg_to_tfexample(
image_data, filename, height, width, seg_data)
else:
example = build_data.image_to_tfexample(
image_data, filename, height, width)
tfrecord_writer.write(example.SerializeToString())
sys.stdout.write('\n')
sys.stdout.flush()
def _convert_dataset(dataset_split, dataset_dir, dataset_label_dir):
image_reader = build_data.ImageReader('jpeg', channels=3)
image_names = os.listdir(FLAGS.train_image_folder)
image_names[:] = [x for x in image_names if not x.startswith('.')]
image_names.sort()
output_filename = '%s_%s.tfrecord' % ('Carvana', dataset_split)
output_filename = os.path.join(FLAGS.build_datadir, output_filename)
with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer:
total = 0
for idx, image_name in enumerate(image_names):
if idx % 5 == 0:
print('total', total, 'file(s), process', idx, 'file(s).')
data_path = os.path.join(dataset_dir, image_name)
label_path = os.path.join(dataset_label_dir,
image_name[:-4] + '_mask.png')
image_data = tf.gfile.GFile(data_path, 'rb').read()
seg_data = tf.gfile.GFile(label_path, 'rb').read()
height, width = image_reader.read_image_dims(image_data)
try:
tf_example = build_data.image_seg_to_tfexample(
image_data, image_name, height, width, seg_data)
if tf_example is not None:
tfrecord_writer.write(tf_example.SerializeToString())
except ValueError:
tf.logging.warning('Invalid example:', image_name,
', ignorig.')
total += 1
print('total', total, 'file(s), process', idx, 'file(s).')
def write_tfexample(image_name, image_reader, label_reader, train_df,
tfrecord_writer):
# Read the image.
image_filename = os.path.join(FLAGS.train_folder, image_name)
image_data = tf.gfile.FastGFile(image_filename, 'rb').read()
height, width = image_reader.read_image_dims(image_data)
# Read the semantic segmentation annotation.
mask_list = train_df['EncodedPixels'][train_df['ImageId'] ==
image_name].tolist()
seg_mask = np.zeros((768, 768, 1))
for item in mask_list:
rle_list = str(item).split()
tmp_mask = rle_to_mask(rle_list, (768, 768))
seg_mask[:, :, 0] += tmp_mask
seg_filename = os.path.join(FLAGS.seg_folder, image_name[:-4] + '.png')
cv2.imwrite(seg_filename, seg_mask)
seg_data = tf.gfile.FastGFile(seg_filename, 'rb').read()
seg_height, seg_width = label_reader.read_image_dims(seg_data)
if height != seg_height or width != seg_width:
raise RuntimeError('Shape mismatched between image and label.')
# Convert to tf example.
example = build_data.image_seg_to_tfexample(image_data, image_name[:-4],
height, width, seg_data)
tfrecord_writer.write(example.SerializeToString())
def _convert_dataset(dataset_split, dataset_dir, dataset_label_dir):
"""Converts the moles_detector dataset into tfrecord format.
Args:
dataset_split: Dataset split (e.g., train, val).
dataset_dir: Dir in which the dataset locates.
dataset_label_dir: Dir in which the annotations locates.
Raises:
RuntimeError: If loaded image and label have different shape.
"""
max_depth = 4
img_names = []
for depth in range(0, max_depth):
img_names += tf.gfile.Glob(
os.path.join(dataset_dir + '/*' * depth, '*.jpg'))
random.shuffle(img_names)
seg_names = []
for f in img_names:
# get the filename without the extension
basename = os.path.basename(f).split('.')[0]
# cover its corresponding *_seg.png
seg = os.path.join(dataset_label_dir, basename + '.png')
seg_names.append(seg)
num_images = len(img_names)
num_per_shard = int(math.ceil(num_images / _NUM_SHARDS))
image_reader = build_data.ImageReader('jpg', channels=3)
label_reader = build_data.ImageReader('png', channels=1)
for shard_id in range(_NUM_SHARDS):
output_filename = os.path.join(
FLAGS.output_dir, '%s-%05d-of-%05d.tfrecord' %
(dataset_split, shard_id, _NUM_SHARDS))
with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer:
start_idx = shard_id * num_per_shard
end_idx = min((shard_id + 1) * num_per_shard, num_images)
for i in range(start_idx, end_idx):
sys.stdout.write('\r>> Converting image %d/%d shard %d' %
(i + 1, num_images, shard_id))
sys.stdout.flush()
# Read the image.
image_filename = img_names[i]
image_data = tf.gfile.FastGFile(image_filename, 'rb').read()
height, width = image_reader.read_image_dims(image_data)
# Read the semantic segmentation annotation.
seg_filename = seg_names[i]
seg_data = tf.gfile.FastGFile(seg_filename, 'rb').read()
seg_height, seg_width = label_reader.read_image_dims(seg_data)
if height != seg_height or width != seg_width:
raise RuntimeError(
'Shape mismatched between image and label.')
# Convert to tf example.
example = build_data.image_seg_to_tfexample(
image_data, img_names[i], height, width, seg_data)
tfrecord_writer.write(example.SerializeToString())
sys.stdout.write('\n')
sys.stdout.flush()
def _convert_dataset():
"""Converts the specified dataset split to TFRecord format.
Raises:
RuntimeError: If loaded image and label have different shape.
"""
dataset = 'DUTS-TR-hflip'
sys.stdout.write('Processing ' + dataset)
filenames_ext = os.listdir(FLAGS.image_folder)
print("filenames_ext")
print(filenames_ext)
filenames = []
for data in filenames_ext:
filenames.append(os.path.splitext(data)[0])
print("filenames")
print(filenames)
num_images = len(filenames)
num_per_shard = int(math.ceil(num_images / _NUM_SHARDS))
image_reader = build_data.ImageReader('jpg', channels=3)
label_reader = build_data.ImageReader('png', channels=1)
for shard_id in range(_NUM_SHARDS):
output_filename = os.path.join(
FLAGS.output_dir,
'%s-%05d-of-%05d.tfrecord' % (dataset, shard_id, _NUM_SHARDS))
with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer:
start_idx = shard_id * num_per_shard
end_idx = min((shard_id + 1) * num_per_shard, num_images)
for i in range(start_idx, end_idx):
sys.stdout.write('\r>> Converting image %d/%d shard %d' %
(i + 1, len(filenames), shard_id))
sys.stdout.flush()
# Read the image.
image_filename = os.path.join(
FLAGS.image_folder,
filenames[i] + '.' + FLAGS.image_format)
image_data = tf.gfile.GFile(image_filename, 'rb').read()
height, width = image_reader.read_image_dims(image_data)
# Read the semantic segmentation annotation.
seg_filename = os.path.join(
FLAGS.semantic_segmentation_folder,
filenames[i] + '.' + FLAGS.label_format)
seg_data = tf.gfile.GFile(seg_filename, 'rb').read()
seg_height, seg_width = label_reader.read_image_dims(seg_data)
if height != seg_height or width != seg_width:
raise RuntimeError(
'Shape mismatched between image and label.')
# Convert to tf example.
example = build_data.image_seg_to_tfexample(
image_data, filenames[i], height, width, seg_data)
tfrecord_writer.write(example.SerializeToString())
sys.stdout.write('\n')
sys.stdout.flush()
def _convert_dataset(dataset_split):
"""Converts the specified dataset split to TFRecord format.
Args:
dataset_split: The dataset split (e.g., train, test).
Raises:
RuntimeError: If loaded image and label have different shape.
"""
dataset = os.path.basename(dataset_split)[:-4]
sys.stdout.write('Processing ' + dataset)
subdirectories = [x.strip('\n') for x in open(dataset_split, 'r')]
_NUM_SHARDS = len(subdirectories)
image_reader = build_data.ImageReader('jpg', channels=3)
label_reader = build_data.ImageReader('png', channels=1)
for shard_id in range(_NUM_SHARDS):
subdirectory = subdirectories[shard_id]
image_names = os.listdir(FLAGS.image_folder + subdirectory)
filenames = [
subdirectory + '/' + image_name[:image_name.rindex('.')]
for image_name in image_names
]
filenames.sort()
output_filename = os.path.join(
FLAGS.output_dir,
'%s-%05d-of-%05d.tfrecord' % (dataset, shard_id, _NUM_SHARDS))
with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer:
for i in range(len(filenames)):
sys.stdout.write('\r>> Converting subdirectory %s shard %d' %
(subdirectory, shard_id))
sys.stdout.flush()
# Read the image.
image_filename = os.path.join(
FLAGS.image_folder,
filenames[i] + '.' + FLAGS.image_format)
image_data = tf.gfile.FastGFile(image_filename, 'rb').read()
height, width = image_reader.read_image_dims(image_data)
# Read the semantic segmentation annotation.
seg_filename = os.path.join(
FLAGS.semantic_segmentation_folder,
filenames[i] + '.' + FLAGS.label_format)
seg_data = tf.gfile.FastGFile(seg_filename, 'rb').read()
seg_height, seg_width = label_reader.read_image_dims(seg_data)
if height != seg_height or width != seg_width:
raise RuntimeError(
'Shape mismatched between image and label.')
# Convert to tf example.
example = build_data.image_seg_to_tfexample(
image_data, filenames[i], height, width, seg_data)
tfrecord_writer.write(example.SerializeToString())
sys.stdout.write('\n')
sys.stdout.flush()
def _convert_dataset(dataset_split):
"""Converts the specified dataset split to TFRecord format.
Args:
dataset_split: The dataset split (e.g., train, test).
Raises:
RuntimeError: If loaded image and label have different shape.
"""
dataset = os.path.basename(dataset_split)[:-4]
sys.stdout.write('Processing ' + dataset)
filenames = [x.strip('\n').split(",")[0] for x in open(dataset_split, 'r')]
segs = [x.strip('\n').split(",")[2:] for x in open(dataset_split, 'r')]
filenames = filenames[1:]
segs = segs[1:]
num_images = len(filenames)
num_per_shard = int(math.ceil(num_images / float(_NUM_SHARDS)))
image_reader = build_data.ImageReader('jpeg', channels=3)
label_reader = build_data.ImageReader('png', channels=1)
for shard_id in range(_NUM_SHARDS):
output_filename = os.path.join(
FLAGS.output_dir,
'%s-%05d-of-%05d.tfrecord' % (dataset, shard_id, _NUM_SHARDS))
with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer:
start_idx = shard_id * num_per_shard
end_idx = min((shard_id + 1) * num_per_shard, num_images)
for i in range(start_idx, end_idx):
sys.stdout.write('\r>> Converting image %d/%d shard %d ' % (
i + 1, len(filenames), shard_id))
sys.stdout.flush()
# Read the image.
image_filename = os.path.join(
FLAGS.image_folder, filenames[i])
image_data = tf.gfile.FastGFile(image_filename, 'rb').read()
height,width = image_reader.read_image_dims(image_data)
seg_label = segs[i]
seg_data = np.zeros([height,width,1])
print(height, width)
for kp_index,kp_info in enumerate(seg_label):
items = [int(x) for x in kp_info.split("_")]
if items[2] >= 0:
seg_data[min(items[0],height-1),min(items[1],width-1),[0]] = kp_index+1
#seg_data = Image.fromarray(seg_data)
seg_data = label_reader.encode_image(seg_data)
example = build_data.image_seg_to_tfexample(
image_data, filenames[i], height, width, seg_data)
tfrecord_writer.write(example.SerializeToString())
sys.stdout.write('\n')
sys.stdout.flush()
def _convert_dataset(dataset_split, dataset_dir, dataset_label_dir):
"""Converts the ADE20k dataset into into tfrecord format.
Args:
dataset_split: Dataset split (e.g., train, val).
dataset_dir: Dir in which the dataset locates.
dataset_label_dir: Dir in which the annotations locates.
Raises:
RuntimeError: If loaded image and label have different shape.
"""
img_names = tf.gfile.Glob(os.path.join(dataset_dir, '*.jpg'))
random.shuffle(img_names)
seg_names = []
for f in img_names:
# get the filename without the extension
basename = os.path.basename(f).split('.')[0]
# cover its corresponding *_seg.png
seg = os.path.join(dataset_label_dir, basename+'.png')
seg_names.append(seg)
num_images = len(img_names)
num_per_shard = int(math.ceil(num_images / float(_NUM_SHARDS)))
image_reader = build_data.ImageReader('jpeg', channels=3)
label_reader = build_data.ImageReader('png', channels=1)
for shard_id in range(_NUM_SHARDS):
output_filename = os.path.join(
FLAGS.output_dir,
'%s-%05d-of-%05d.tfrecord' % (dataset_split, shard_id, _NUM_SHARDS))
with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer:
start_idx = shard_id * num_per_shard
end_idx = min((shard_id + 1) * num_per_shard, num_images)
for i in range(start_idx, end_idx):
sys.stdout.write('\r>> Converting image %d/%d shard %d' % (
i + 1, num_images, shard_id))
sys.stdout.flush()
# Read the image.
image_filename = img_names[i]
image_data = tf.gfile.FastGFile(image_filename, 'r').read()
height, width = image_reader.read_image_dims(image_data)
# Read the semantic segmentation annotation.
seg_filename = seg_names[i]
seg_data = tf.gfile.FastGFile(seg_filename, 'r').read()
seg_height, seg_width = label_reader.read_image_dims(seg_data)
if height != seg_height or width != seg_width:
raise RuntimeError('Shape mismatched between image and label.')
# Convert to tf example.
example = build_data.image_seg_to_tfexample(
image_data, img_names[i], height, width, seg_data)
tfrecord_writer.write(example.SerializeToString())
sys.stdout.write('\n')
sys.stdout.flush()
def _convert_dataset(dataset_split, image_names, labels_df):
"""Converts the ADE20k dataset into into tfrecord format.
Args:
dataset_split: Dataset split (e.g., train, val).
image_ids: ...
labels_df: ...
Raises:
RuntimeError: If loaded image and label have different shape.
"""
num_images = len(image_names)
num_per_shard = int(math.ceil(num_images / float(_NUM_SHARDS)))
image_class_id_to_rle_mask = dict(
zip(labels_df.ImageId_ClassId, labels_df.EncodedPixels))
image_reader = build_data.ImageReader('jpeg', channels=3)
# label_reader = build_data.ImageReader('png', channels=1)
for shard_id in range(_NUM_SHARDS):
output_filename = os.path.join(
FLAGS.output_dir, '%s-%05d-of-%05d.tfrecord' %
(dataset_split, shard_id, _NUM_SHARDS))
with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer:
start_idx = shard_id * num_per_shard
end_idx = min((shard_id + 1) * num_per_shard, num_images)
for i in range(start_idx, end_idx):
sys.stdout.write('\r>> Converting image %d/%d shard %d' %
(i + 1, num_images, shard_id))
sys.stdout.flush()
# Read the image.
image_name = image_names[i]
image_data = tf.gfile.FastGFile(image_name, 'rb').read()
height, width = image_reader.read_image_dims(image_data)
# Read the semantic segmentation annotation.
image_id = image_name.split('/')[-1].split('.')[0]
rle_masks = [
image_class_id_to_rle_mask['{}.jpg_{}'.format(
image_id, i + 1)] for i in range(4)
]
masks = [
image_utils.rle_to_mask(rle_mask, height, width)
for rle_mask in rle_masks
]
mask = masks_to_mask(masks)
mask_data = image_utils.numpy_to_bytes(mask, 'png')
# Convert to tf example.
example = build_data.image_seg_to_tfexample(
image_data, image_name, height, width, mask_data)
tfrecord_writer.write(example.SerializeToString())
sys.stdout.write('\n')
sys.stdout.flush()
def _convert_dataset(dataset_split):
"""Converts the specified dataset split to TFRecord format.
Args:
dataset_split: The dataset split (e.g., train, test).
Raises:
RuntimeError: If loaded image and label have different shape.
"""
dataset = os.path.basename(dataset_split)[:-4]
sys.stdout.write('Processing ' + dataset)
filenames = [x.strip('\n') for x in open(dataset_split, 'r')]
num_images = len(filenames)
print('\n num images')
print(num_images)
num_per_shard = int(math.ceil(num_images / float(_NUM_SHARDS)))
image_reader = build_data.ImageReader('png', channels=3)
label_reader = build_data.ImageReader('png', channels=1)
for shard_id in range(_NUM_SHARDS):
output_filename = os.path.join(
FLAGS.output_dir,
'%s-%05d-of-%05d.tfrecord' % (dataset, shard_id, _NUM_SHARDS))
print('output_filename')
print(output_filename)
with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer:
start_idx = shard_id * num_per_shard
end_idx = min((shard_id + 1) * num_per_shard, num_images)
for i in range(start_idx, end_idx):
sys.stdout.write('\r>> Converting image %d/%d shard %d' % (
i + 1, len(filenames), shard_id))
sys.stdout.flush()
# Read the image.
image_filename = os.path.join(
FLAGS.image_folder, filenames[i] + '.' + FLAGS.image_format)
image_data = tf.gfile.FastGFile(image_filename, 'rb').read()
height, width = image_reader.read_image_dims(image_data)
print('\n ---image dims')
print(height)
print(width)
# Read the semantic segmentation annotation.
seg_filename = os.path.join(
FLAGS.semantic_segmentation_folder,
filenames[i] + '.' + FLAGS.label_format)
seg_data = tf.gfile.FastGFile(seg_filename, 'rb').read()
seg_height, seg_width = label_reader.read_image_dims(seg_data)
if height != seg_height or width != seg_width:
raise RuntimeError('Shape mismatched between image and label.')
# Convert to tf example.
example = build_data.image_seg_to_tfexample(
image_data, filenames[i], height, width, seg_data)
tfrecord_writer.write(example.SerializeToString())
sys.stdout.write('\n')
sys.stdout.flush()
def write_tfexample(image_name, image_reader, train_df, tfrecord_writer,
label):
# Read the image.
image_filename = os.path.join(FLAGS.train_folder, image_name)
image_data = tf.gfile.FastGFile(image_filename, 'rb').read()
height, width = image_reader.read_image_dims(image_data)
# Convert to tf example.
example = build_data.image_seg_to_tfexample(image_data, image_name[:-4],
height, width, label)
tfrecord_writer.write(example.SerializeToString())
def create_tfrecords(src_files, file_ids, n_shards, sub_seq, use_tif,
output_dir):
if use_tif:
image_reader = build_data.ImageReader('png', channels=1)
else:
image_reader = build_data.ImageReader('jpeg', channels=1)
label_reader = build_data.ImageReader('png', channels=1)
n_images = len(src_files)
n_per_shard = int(math.ceil(n_images / float(n_shards)))
os.makedirs(output_dir, exist_ok=True)
print('Creating {} shards with {} images ({} per shard)'.format(
n_shards, n_images, n_per_shard))
for shard_id in range(n_shards):
output_file_path = os.path.join(
output_dir,
'{:s}-{:05d}-of-{:05d}.tfrecord'.format(sub_seq, shard_id,
n_shards))
with tf.python_io.TFRecordWriter(output_file_path) as tfrecord_writer:
start_idx = shard_id * n_per_shard
end_idx = min((shard_id + 1) * n_per_shard, n_images)
for img_id in tqdm(range(start_idx, end_idx), ncols=50):
img_src_file = src_files[img_id]
img_src_file_id, seq_name, seg_src_path, img_src_path = file_ids[
img_src_file]
image_data = tf.gfile.FastGFile(img_src_path, 'rb').read()
height, width = image_reader.read_image_dims(image_data)
if seg_src_path is not None:
seg_data = tf.gfile.FastGFile(seg_src_path, 'rb').read()
seg_height, seg_width = label_reader.read_image_dims(
seg_data)
if height != seg_height or width != seg_width:
raise RuntimeError(
'Shape mismatch found between image and label')
else:
seg_data = None
# Convert to tf example.
example = build_data.image_seg_to_tfexample(
image_data, img_src_path, height, width, seg_data)
tfrecord_writer.write(example.SerializeToString())
def _convert_dataset(dataset_split):
dataset = os.path.basename(dataset_split)[:-4]
sys.stdout.write('Processing ' + dataset)
filenames = [x.strip('\n') for x in open(dataset_split, 'r')]
print(filenames)
num_images = len(filenames)
num_per_shard = int(math.ceil(num_images / float(_NUM_SHARDS)))
image_reader = build_data.ImageReader('jpeg', channels=3)
label_reader = build_data.ImageReader('jpg', channels=1)
for shard_id in range(_NUM_SHARDS):
output_filename = os.path.join(
FLAGS.output_dir,
'%s-%05d-of-%05d.tfrecord' % (dataset, shard_id, _NUM_SHARDS))
with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer:
start_idx = shard_id * num_per_shard
end_idx = min((shard_id + 1) * num_per_shard, num_images)
for i in range(start_idx, end_idx):
sys.stdout.write('\r>> Converting image %d/%d shard %d' % (
i + 1, len(filenames), shard_id))
print("hell")
sys.stdout.flush()
# Read the image.
image_filename = os.path.join(
FLAGS.image_folder, filenames[i])# + '.' + FLAGS.image_format)
print(image_filename)
image_data = tf.gfile.GFile(image_filename, 'rb').read()
print("what")
height, width = image_reader.read_image_dims(image_data)
print("the")
# Read the semantic segmentation annotation.
seg_filename = os.path.join(
FLAGS.semantic_segmentation_folder,
filenames[i])# + '.' + FLAGS.label_format)
seg_data = tf.gfile.GFile(seg_filename, 'rb').read()
seg_height, seg_width = label_reader.read_image_dims(seg_data)
if height != seg_height or width != seg_width:
raise RuntimeError('Shape mismatched between image and label.')
# Convert to tf example.
example = build_data.image_seg_to_tfexample(
image_data, filenames[i], height, width, seg_data)
B = tf.io.TFRecordWriter('./tfrecord/storage.tfrecord')
B.write(example.SerializeToString())
print("me")
sys.stdout.write('\n')
sys.stdout.flush()
def _convert_dataset(dataset_split):
"""Converts the specified dataset split to TFRecord format.
Args:
dataset_split: The dataset split (e.g., train, test).
Raises:
RuntimeError: If loaded image and label have different shape.
"""
# val.txt
dataset = os.path.basename(dataset_split)[:-4]
input('dataset: %s' % dataset)
sys.stdout.write('Processing ' + dataset)
input('after processing')
filenames = [x.strip('\n') for x in open(dataset_split, 'r')]
num_images = len(filenames)
num_per_shard = int(math.ceil(num_images / float(_NUM_SHARDS)))
input('before image reader')
image_reader = build_data.ImageReader('jpeg', channels=3)
# label_reader = build_data.ImageReader('png', channels=1)
label_reader = build_data.ImageReader('jpeg', channels=3)
for shard_id in range(_NUM_SHARDS):
output_filename = os.path.join(
FLAGS.output_dir,
'%s-%05d-of-%05d.tfrecord' % (dataset, shard_id, _NUM_SHARDS))
input('before writer')
with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer:
start_idx = shard_id * num_per_shard
end_idx = min((shard_id + 1) * num_per_shard, num_images)
for i in range(start_idx, end_idx):
sys.stdout.write('\r>> Converting image %d/%d shard %d' % (
i + 1, len(filenames), shard_id))
sys.stdout.flush()
# Read the image.
image_filename = os.path.join(
FLAGS.image_folder, filenames[i] + '.' + FLAGS.image_format)
image_data = tf.gfile.FastGFile(image_filename, 'rb').read()
height, width = image_reader.read_image_dims(image_data)
seg_data = image_data
# Convert to tf example.
example = build_data.image_seg_to_tfexample(
image_data, filenames[i], height, width, seg_data)
tfrecord_writer.write(example.SerializeToString())
sys.stdout.write('\n')
sys.stdout.flush()
def _convert_dataset(dataset_split):
"""Converts the specified dataset split to TFRecord format.
Args:
dataset_split: The dataset split (e.g., train, val).
Raises:
RuntimeError: If loaded image and label have different shape, or if the
image file with specified postfix could not be found.
"""
image_files = _get_files('image', dataset_split)
label_files = _get_files('label', dataset_split)
num_images = len(image_files)
num_per_shard = int(math.ceil(num_images / float(_NUM_SHARDS)))
image_reader = build_data.ImageReader('png', channels=3)
label_reader = build_data.ImageReader('png', channels=1)
for shard_id in range(_NUM_SHARDS):
shard_filename = '%s-%05d-of-%05d.tfrecord' % (
dataset_split, shard_id, _NUM_SHARDS)
output_filename = os.path.join(FLAGS.output_dir, shard_filename)
with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer:
start_idx = shard_id * num_per_shard
end_idx = min((shard_id + 1) * num_per_shard, num_images)
for i in range(start_idx, end_idx):
sys.stdout.write('\r>> Converting image %d/%d shard %d' % (
i + 1, num_images, shard_id))
sys.stdout.flush()
# Read the image.
image_data = tf.gfile.FastGFile(image_files[i], 'r').read()
height, width = image_reader.read_image_dims(image_data)
# Read the semantic segmentation annotation.
seg_data = tf.gfile.FastGFile(label_files[i], 'r').read()
seg_height, seg_width = label_reader.read_image_dims(seg_data)
if height != seg_height or width != seg_width:
raise RuntimeError('Shape mismatched between image and label.')
# Convert to tf example.
re_match = _IMAGE_FILENAME_RE.search(image_files[i])
if re_match is None:
raise RuntimeError('Invalid image filename: ' + image_files[i])
filename = os.path.basename(re_match.group(1))
example = build_data.image_seg_to_tfexample(
image_data, filename, height, width, seg_data)
tfrecord_writer.write(example.SerializeToString())
sys.stdout.write('\n')
sys.stdout.flush()
def create_tfrecords(img_src_files, seg_src_files, n_shards, db_split,
output_dir):
image_reader = ImageReader('jpeg', channels=1)
label_reader = ImageReader('png', channels=1)
n_images = len(img_src_files)
n_per_shard = int(math.ceil(n_images / float(n_shards)))
os.makedirs(output_dir, exist_ok=True)
print('Creating {} shards with {} images ({} per shard)'.format(
n_shards, n_images, n_per_shard))
for shard_id in range(n_shards):
output_file_path = os.path.join(
output_dir,
'{:s}-{:05d}-of-{:05d}.tfrecord'.format(db_split, shard_id,
n_shards))
with tf.python_io.TFRecordWriter(output_file_path) as tfrecord_writer:
start_idx = shard_id * n_per_shard
end_idx = min((shard_id + 1) * n_per_shard, n_images)
for img_id in tqdm(range(start_idx, end_idx), ncols=50):
img_src_path = img_src_files[img_id]
image_data = tf.gfile.FastGFile(img_src_path, 'rb').read()
height, width = image_reader.read_image_dims(image_data)
if seg_src_files is not None and seg_src_files:
seg_src_path = seg_src_files[img_id]
seg_data = tf.gfile.FastGFile(seg_src_path, 'rb').read()
seg_height, seg_width = label_reader.read_image_dims(
seg_data)
assert height == seg_height and width == seg_width, 'Shape mismatch found between image and label'
else:
seg_data = None
# Convert to tf example.
example = image_seg_to_tfexample(image_data, img_src_path,
height, width, seg_data)
tfrecord_writer.write(example.SerializeToString())
def convert_dataset(dataset, names, num_shared=4):
sys.stdout.write('Processing ' + dataset)
sys.stdout.write('\n')
sys.stdout.flush()
num = len(names)
num_per_shard = int(math.ceil(num / float(num_shared)))
image_reader = build_data.ImageReader('jpg', channels=3)
label_reader = build_data.ImageReader('png', channels=1)
for shard_id in range(num_shared):
output_filename = os.path.join(
dataset_folder, 'tfrecord',
'%s-%05d-of-%05d.tfrecord' % (dataset, shard_id, num_shared))
with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer:
start_idx = shard_id * num_per_shard
end_idx = min((shard_id + 1) * num_per_shard, num)
for i in range(start_idx, end_idx):
sys.stdout.write('\r>> Converting image %d/%d shard %d' %
(i + 1, num, shard_id))
sys.stdout.flush()
# Read the image.
image_filename = os.path.join(dataset_folder, 'train',
names[i] + data_format)
image_data = tf.gfile.FastGFile(image_filename, 'rb').read()
height, width = image_reader.read_image_dims(image_data)
# Read the semantic segmentation annotation.
seg_filename = os.path.join(dataset_folder, 'seg',
names[i] + seg_format)
seg_data = tf.gfile.FastGFile(seg_filename, 'rb').read()
seg_height, seg_width = label_reader.read_image_dims(seg_data)
if height != seg_height or width != seg_width:
raise RuntimeError(
'Shape mismatched between image and label.')
# Convert to tf example.
example = build_data.image_seg_to_tfexample(
image_data, names[i], height, width, seg_data)
tfrecord_writer.write(example.SerializeToString())
sys.stdout.write('\n')
sys.stdout.flush()
def _convert_dataset(dataset_split):
"""Converts the specified dataset split to TFRecord format.
Args:
dataset_split: The dataset split (e.g., train, val).
Raises:
RuntimeError: If loaded image and label have different shape, or if the
image file with specified postfix could not be found.
"""
image_files = _get_files(FLAGS.image_side, dataset_split)
num_images = len(image_files)
num_per_shard = int(math.ceil(num_images / float(_NUM_SHARDS)))
image_reader = build_data.ImageReader('png', channels=3)
for shard_id in range(_NUM_SHARDS):
shard_filename = '%s-%05d-of-%05d.tfrecord' % (
dataset_split, shard_id, _NUM_SHARDS)
output_filename = os.path.join(FLAGS.output_dir, shard_filename)
with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer:
start_idx = shard_id * num_per_shard
end_idx = min((shard_id + 1) * num_per_shard, num_images)
for i in range(start_idx, end_idx):
sys.stdout.write('\r>> Converting image %d/%d shard %d' % (i + 1, num_images, shard_id))
sys.stdout.flush()
# Read the image.
image_data = tf.gfile.FastGFile(image_files[i], 'rb').read()
height, width = image_reader.read_image_dims(image_data)
# Convert to tf example.
print(image_files[i])
filename = os.path.basename(image_files[i])
example = build_data.image_seg_to_tfexample(
image_data, filename, height, width)
tfrecord_writer.write(example.SerializeToString())
sys.stdout.write('\n')
sys.stdout.flush()
def _convert_dataset(seq_idx):
"""Converts the specified dataset split to TFRecord format.
Args:
dataset_split: The dataset split (e.g., train, val).
Raises:
RuntimeError: If loaded image and label have different shape, or if the
image file with specified postfix could not be found.
"""
image_files = _get_files('image', seq_idx)
label_files = _get_files('label', seq_idx)
image_reader = build_data.ImageReader('png', channels=3)
label_reader = build_data.ImageReader('png', channels=1)
shard_filename = 'seq-%05d-of-%05d.tfrecord' % (0, 1)
output_filename = os.path.join(usr_dir, FLAGS.cityscapes_root,
FLAGS.output_dir, shard_filename)
with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer:
for i in xrange(len(image_files)):
# Read the image.
image_data = tf.gfile.FastGFile(image_files[i], 'rb').read()
height, width = image_reader.read_image_dims(image_data)
# Read the semantic segmentation annotation.
seg_data = tf.gfile.FastGFile(label_files[i], 'rb').read()
seg_height, seg_width = label_reader.read_image_dims(seg_data)
if height != seg_height or width != seg_width:
raise RuntimeError('Shape mismatched between image and label.')
# Convert to tf example.
re_match = _IMAGE_FILENAME_RE.search(image_files[i])
if re_match is None:
raise RuntimeError('Invalid image filename: ' + image_files[i])
filename = os.path.basename(re_match.group(1))
example = build_data.image_seg_to_tfexample(
image_data, filename, height, width, seg_data)
tfrecord_writer.write(example.SerializeToString())
def _convert_dataset(dataset_split):
"""Converts the specified dataset split to TFRecord format.
Args:
dataset_split: The dataset split (e.g., train, test).
Raises:
RuntimeError: If loaded image and label have different shape.
"""
dataset = os.path.basename(dataset_split)[:-4]
sys.stdout.write('Processing ' + dataset)
filenames = [x.strip('\n') for x in open(dataset_split, 'r')]
#print(filenames)
#print("FOI O FILENAMES _convert_dataset", dataset_split)
split = dataset_split.split('/')[-1].split('.')[0]
positive_masks = []
negative_masks = []
for name in filenames:
seg_filename = os.path.join(
FLAGS.semantic_segmentation_folder,
name + '.' + FLAGS.label_format)
seg = misc.imread(seg_filename)
if len(seg[seg>0]) > 0:
positive_masks.append(name)
else:
negative_masks.append(name)
#print(positive_masks, len(positive_masks))
idx = [negative_masks, positive_masks]
#num_per_shard = int(math.ceil(num_images / float(_NUM_SHARDS)))
# num_per_shard depende da quantidade de imagens por classe.
image_reader = build_data.ImageReader('png', channels=3)
label_reader = build_data.ImageReader('png', channels=1)
for shard_id in range(len(['negative', 'positive'])):
output_filename = os.path.join(
FLAGS.output_dir,
'%s-%05d-of-%05d.tfrecord' % (dataset, shard_id, _NUM_SHARDS))
with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer:
start_idx = 0
end_idx = len(idx[shard_id])
for i in range(start_idx, end_idx):
sys.stdout.write('\r>> Converting image %d/%d shard %d' % (
i + 1, len(idx[shard_id]), shard_id))
sys.stdout.flush()
# Read the image.
image_filename = os.path.join(
FLAGS.image_folder, idx[shard_id][i] + '.' + FLAGS.image_format)
image_data = tf.gfile.FastGFile(image_filename, 'rb').read()
height, width = image_reader.read_image_dims(image_data)
# Read the semantic segmentation annotation.
seg_filename = os.path.join(
FLAGS.semantic_segmentation_folder,
idx[shard_id][i] + '.' + FLAGS.label_format)
seg_data = tf.gfile.FastGFile(seg_filename, 'rb').read()
seg_height, seg_width = label_reader.read_image_dims(seg_data)
if height != seg_height or width != seg_width:
raise RuntimeError('Shape mismatched between image and label.')
# Convert to tf example.
example = build_data.image_seg_to_tfexample(
image_data, idx[shard_id][i], height, width, seg_data)
tfrecord_writer.write(example.SerializeToString())
sys.stdout.write('\n')
sys.stdout.flush()
def _convert_dataset(dataset_split, dataset):
"""Converts the specified dataset split to TFRecord format.
Raises:
RuntimeError: If loaded image and label have different shape, or if the
image file with specified postfix could not be found.
"""
img_dir_name = FLAGS.img_dir_name
labels_dir_name = FLAGS.trainIDs_dir_name
output_dir = FLAGS.output_dir
image_files = dataset[0]
label_files = dataset[1]
if not (len(image_files) == len(label_files)):
raise RuntimeError('Length mismatch image and label.')
num_images = len(image_files)
num_per_shard = int(math.ceil(num_images / float(_NUM_SHARDS)))
image_reader = build_data.ImageReader('png', channels=3)
label_reader = build_data.ImageReader('png', channels=1)
for shard_id in range(_NUM_SHARDS):
shard_filename = '%s-%05d-of-%05d.tfrecord' % (dataset_split, shard_id,
_NUM_SHARDS)
output_filename = os.path.join(output_dir, shard_filename)
with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer:
start_idx = shard_id * num_per_shard
end_idx = min((shard_id + 1) * num_per_shard, num_images)
for i in range(start_idx, end_idx):
sys.stdout.write('\r>> Converting image %d/%d shard %d' %
(i + 1, num_images, shard_id))
sys.stdout.flush()
# Read the image.
image_data = tf.gfile.FastGFile(img_dir_name + image_files[i],
'rb').read()
height, width = image_reader.read_image_dims(image_data)
# Read the semantic segmentation annotation.
seg_data = tf.gfile.FastGFile(labels_dir_name + label_files[i],
'rb').read()
seg_height, seg_width = label_reader.read_image_dims(seg_data)
if height != seg_height or width != seg_width:
print(
"Shape mismatched between image and label. height. Ignoring."
)
continue
raise RuntimeError(
'Shape mismatched between image and label. height : ',
height, ' seg_height: ', seg_height, ' width: ', width,
' seg_width: ', seg_width, ' \nlabel_files[i]: ',
label_files[i], ' image_files[i]: ', image_files[i])
# Convert to tf example.
if not (image_files[i] == label_files[i]):
raise RuntimeError('image file name : ' + image_files[i] +
' is not equal to label file name : ' +
label_files[i])
filename = os.path.basename(image_files[i])
example = build_data.image_seg_to_tfexample(
image_data, filename, height, width, seg_data)
tfrecord_writer.write(example.SerializeToString())
sys.stdout.write('\n')
sys.stdout.flush()
def _convert_dataset(dataset_patches, train=True):
"""Converts the specified dataset split to TFRecord format.
Args:
dataset_split: The patches of remote sensing dataset (e.g., train, test).
train: True means that one patch contains both a remote sensing image and a label image,
False means that one patch only contains a Remote sensing image
Raises:
RuntimeError: If loaded image and label have different shape.
"""
# notes: http://warmspringwinds.github.io/tensorflow/tf-slim/2016/12/21/tfrecords-guide/
# notes: https://gist.github.com/swyoon/8185b3dcf08ec728fb22b99016dd533f
num_images = len(dataset_patches)
num_per_shard = int(math.ceil(num_images / float(_NUM_SHARDS)))
# use the first image name as dataset_name
org_img = dataset_patches[0][0].org_img
dataset_name = os.path.splitext(os.path.basename(org_img))[0]
if train:
for idx in range(num_images):
img_patch, gt_patch = dataset_patches[idx]
image_data = read_patch(img_patch)
label_data = read_patch(
gt_patch) #build_data.ImageReader('png', channels=1)
for shard_id in range(_NUM_SHARDS):
output_filename = os.path.join(
FLAGS.output_dir, '%s-%05d-of-%05d.tfrecord' %
(dataset_name, shard_id, _NUM_SHARDS))
with tf.python_io.TFRecordWriter(
output_filename) as tfrecord_writer:
start_idx = shard_id * num_per_shard
end_idx = min((shard_id + 1) * num_per_shard, num_images)
for i in range(start_idx, end_idx):
sys.stdout.write(
'\r>> Converting image %d/%d shard %d' %
(i + 1, num_images, shard_id))
sys.stdout.flush()
# Read the image.
org_img = img_patch.org_img
file_name = os.path.splitext(
os.path.basename(org_img))[0] + '_' + str(i)
# image_filename = os.path.join(
# FLAGS.image_folder, filenames[i] + '.' + FLAGS.image_format)
# image_data = tf.gfile.FastGFile(image_filename, 'r').read()
# height, width = image_reader.read_image_dims(image_data)
image_shape = image_data.shape
height, width = image_shape[1], image_shape[2]
# # Read the semantic segmentation annotation.
# seg_filename = os.path.join(
# FLAGS.semantic_segmentation_folder,
# filenames[i] + '.' + FLAGS.label_format)
# seg_data = tf.gfile.FastGFile(seg_filename, 'r').read()
label_shape = label_data.shape
label_height, label_width = image_shape[
1], image_shape[2]
# seg_height, seg_width = label_reader.read_image_dims(seg_data)
if height != label_height or width != label_width:
raise RuntimeError(
'Shape mismatched between image and label.')
# Convert to tf example.
example = build_data.image_seg_to_tfexample(
image_data, file_name, height, width, label_data)
tfrecord_writer.write(example.SerializeToString())
sys.stdout.write('\n')
sys.stdout.flush()
else:
# img_patch = self.patches[idx]
# patch_info = [img_patch.org_img, img_patch.boundary]
# # img_name_noext = os.path.splitext(os.path.basename(img_patch.org_img))[0]+'_'+str(idx)
# img = read_patch(img_patch)
# # img.resize(self.nRow,self.nCol)
# img = img[:, 0:self.nRow, 0:self.nCol]
# img = np.atleast_3d(img).astype(np.float32)
# if (img.max() - img.min()) < 0.01:
# pass
# else:
# img = (img - img.min()) / (img.max() - img.min())
# img = torch.from_numpy(img).float()
# return img, patch_info
# dataset = os.path.basename(dataset_split)[:-4]
# sys.stdout.write('Processing ' + dataset)
pass
for ITL_datasetSplit in ITL_datasetSplits:
ITL_dataset = os.path.basename(ITL_datasetSplit)[:-4]
print('Processing ' + ITL_dataset)
ITL_filenames = [x.strip('\n') for x in open(ITL_datasetSplit, 'r')]
ITL_numImages = len(ITL_filenames)
ITL_numPerShard = int(math.ceil(ITL_numImages / float(ITL_numShards)))
ITL_imageReader = build_data.ImageReader('jpeg', channels=3)
ITL_labelReader = build_data.ImageReader('png', channels=1)
for ITL_shardId in range(ITL_numShards):
ITL_outputFilename = os.path.join(ITL_outputDir, '%s-%05d-of-%05d.tfrecord' % (ITL_dataset, ITL_shardId, ITL_numShards))
with tf.python_io.TFRecordWriter(ITL_outputFilename) as tfrecord_writer:
ITL_startIdx = ITL_shardId * ITL_numPerShard
ITL_endIdx = min((ITL_shardId + 1) * ITL_numPerShard, ITL_numImages)
for i in range(ITL_startIdx, ITL_endIdx):
print('\r>> Converting image %d/%d shard %d' % (i + 1, len(ITL_filenames), ITL_shardId))
sys.stdout.flush()
ITL_imageFilename = os.path.join(ITL_imageDir, ITL_filenames[i] + '.jpg')
ITL_imageData = tf.gfile.FastGFile(ITL_imageFilename, 'rb').read()
height, width = ITL_imageReader.read_image_dims(ITL_imageData)
ITL_anoFilename = os.path.join(ITL_anoDir, ITL_filenames[i] + '.png')
ITL_anoData = tf.gfile.FastGFile(ITL_anoFilename, 'rb').read()
ano_height, ano_width = ITL_labelReader.read_image_dims(ITL_anoData)
if height != ano_height or width != ano_width:
raise RuntimeError('Shape mismatched between image and label.')
ITL_example = build_data.image_seg_to_tfexample(ITL_imageData, ITL_filenames[i], height, width, ITL_anoData)
tfrecord_writer.write(ITL_example.SerializeToString())
print('\n')
sys.stdout.flush()
def _convert_dataset(dataset_split):
"""Converts the specified dataset split to TFRecord format.
Args:
dataset_split: The dataset split (e.g., train, val, trainval)
Raises:
RuntimeError: If loaded image and label have different shape.
"""
# get the annotation paths.
pattern = os.path.join(FLAGS.annotation_dir, '%s2017' % dataset_split,
'*.png')
ann_list = sorted(glob.glob(pattern))
image_reader = build_data.ImageReader(channels=3)
label_reader = build_data.ImageReader(channels=1)
if not os.path.exists(FLAGS.output_dir):
os.makedirs(FLAGS.output_dir)
num_img = len(ann_list)
num_img_per_shard = math.ceil(num_img / NUM_SHARD)
print('The number of %s image: %d' % (dataset_split, num_img))
for shard_id in range(NUM_SHARD):
output_filename = os.path.join(
FLAGS.output_dir, '%s-%05d-of-%05d.tfrecord' %
(dataset_split, shard_id + 1, NUM_SHARD))
start_idx = shard_id * num_img_per_shard
end_idx = min((shard_id + 1) * num_img_per_shard, num_img)
with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer:
for i in range(start_idx, end_idx):
ann_path = ann_list[i]
img_path = os.path.join(
FLAGS.image_dir,
ann_path.split(os.sep)[-2],
os.path.basename(ann_path).replace('.png', '.jpg'))
sys.stdout.write('\r>> Converting image %d/%d shard %d' %
(i + 1, num_img, shard_id))
sys.stdout.flush()
if not os.path.exists(img_path):
raise ValueError('image {} dont exists'.format(img_path))
# Read the image
img_data = tf.gfile.GFile(img_path, 'rb').read()
height, width = image_reader.read_image_dims(img_data)
ann_data = tf.gfile.GFile(ann_path, 'rb').read()
ann_height, ann_width = label_reader.read_image_dims(ann_data)
if height != ann_height or width != ann_width:
raise RuntimeError(
'Shape mismatched between image and annotation.')
# Convert to tf example.
example = build_data.image_seg_to_tfexample(
os.path.basename(img_path), img_data, height, width,
ann_data)
tfrecord_writer.write(example.SerializeToString())
sys.stdout.write('\n')
sys.stdout.flush()
def _convert_dataset(dataset_split):
"""Converts the specified dataset split to TFRecord format.
Args:
dataset_split: The dataset split (e.g., train, val).
Raises:
RuntimeError: If loaded image and label have different shape.
"""
sys.stdout.write('Processing ' + dataset_split)
if dataset_split == 'train':
seg_base_dir = 'PubLayNet/SegmentationClass/train/'
raw_base_dir = 'PubLayNet/RawImages/train_data/'
else:
seg_base_dir = 'PubLayNet/SegmentationClass/val/'
raw_base_dir = 'PubLayNet/RawImages/val_data/val/'
seg_file_names = [
f for f in glob.glob(seg_base_dir + "**/*.png", recursive=True)
]
raw_file_names = [
f for f in glob.glob(raw_base_dir + "**/*.jpg", recursive=True)
]
raw_name_only = [f.split('/')[-1].split('.')[-2] for f in raw_file_names]
num_images = len(seg_file_names)
num_shards = int(math.ceil(num_images / float(num_docs_per_shard)))
image_reader = build_data.ImageReader('jpeg', channels=3)
label_reader = build_data.ImageReader('png', channels=1)
for shard_id in range(num_shards):
output_filename = os.path.join(
'./PubLayNet/tfrecord',
'%s-%05d-of-%05d.tfrecord' % (dataset_split, shard_id, num_shards))
with tf.io.TFRecordWriter(output_filename) as tfrecord_writer:
start_idx = shard_id * num_docs_per_shard
end_idx = min((shard_id + 1) * num_docs_per_shard, num_images)
for i in range(start_idx, end_idx):
sys.stdout.write('\r>> Converting image %d/%d shard %d' %
(i + 1, num_images, shard_id))
sys.stdout.flush()
# Read the semantic segmentation annotation.
seg_name_only = seg_file_names[i].split('/')[-1].split('.')[-2]
seg_data = tf.io.gfile.GFile(seg_file_names[i], 'rb').read()
'''print('/n', dataset_split, 'seg', seg_file_names[i])
print(dataset_split, 'seg', seg_name_only)'''
seg_height, seg_width = label_reader.read_image_dims(seg_data)
# Read the raw image.
name_ind = raw_name_only.index(seg_name_only)
'''print(dataset_split, 'raw', raw_file_names[name_ind])
print(dataset_split, 'raw', raw_name_only[name_ind])'''
image_data = tf.io.gfile.GFile(raw_file_names[name_ind],
'rb').read()
height, width = image_reader.read_image_dims(image_data)
if height != seg_height or width != seg_width:
#raise RuntimeError('Shape mismatched between image and label.')
print(
'The raw image and segmentation mask do not have the same dimensions.'
)
print('Skipping image {}'.format(seg_name_only))
else:
# Convert to tf example.
example = build_data.image_seg_to_tfexample(
image_data, seg_name_only, height, width, seg_data)
tfrecord_writer.write(example.SerializeToString())
sys.stdout.write('\n')
sys.stdout.flush()
def _convert_dataset(db_name):
"""Converts the specified dataset split to TFRecord format.
Args:
db_name: The dataset split (e.g., train, test).
Raises:
RuntimeError: If loaded image and label have different shape.
"""
output_dir = os.path.join(FLAGS.db_root_dir, FLAGS.output_dir, 'tfrecord')
sys.stdout.write('Processing {}\n\n'.format(db_name))
images = os.path.join(FLAGS.db_root_dir, db_name, 'images',
'*.{}'.format(FLAGS.image_format))
print('Reading images from: {}'.format(images))
image_filenames = glob.glob(images)
if image_filenames is None:
raise SystemError('No images found at {}'.format(images))
if FLAGS.create_dummy_labels:
labels_path = os.path.join(FLAGS.db_root_dir, db_name, 'labels')
if not os.path.isdir(labels_path):
os.makedirs(labels_path)
print('Creating dummy labels at: {}'.format(labels_path))
for image_filename in image_filenames:
image = misc.imread(image_filename)
height, width, _ = image.shape
dummy_label = np.zeros((height, width), dtype=np.uint8)
out_fname = os.path.splitext(
os.path.basename(image_filename))[0] + '.{}'.format(
FLAGS.label_format)
misc.imsave(os.path.join(labels_path, out_fname), dummy_label)
print('Done')
labels = os.path.join(FLAGS.db_root_dir, db_name, 'labels',
'*.{}'.format(FLAGS.label_format))
print('Reading labels from: {}'.format(labels))
seg_filenames = glob.glob(labels)
if seg_filenames is None:
raise SystemError('No labels found at {}'.format(labels))
# filenames = [x.strip('\n') for x in open(dataset_split, 'r')]
num_images = len(image_filenames)
num_labels = len(seg_filenames)
if num_images != num_labels:
raise SystemError(
'Mismatch between image and label file counts: {}, {}'.format(
num_images, num_labels))
num_per_shard = int(math.ceil(num_images / float(_NUM_SHARDS)))
image_reader = build_data.ImageReader('png', channels=3)
label_reader = build_data.ImageReader('png', channels=3)
if not os.path.isdir(output_dir):
os.makedirs(output_dir)
print('Writing tfrecords to: {}'.format(output_dir))
for shard_id in range(_NUM_SHARDS):
output_filename = os.path.join(
output_dir,
'%s-%05d-of-%05d.tfrecord' % (db_name, shard_id, _NUM_SHARDS))
with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer:
start_idx = shard_id * num_per_shard
end_idx = min((shard_id + 1) * num_per_shard, num_images)
for i in range(start_idx, end_idx):
sys.stdout.write('\r>> Converting image %d/%d shard %d' %
(i + 1, num_images, shard_id))
sys.stdout.flush()
image_filename = image_filenames[i]
f1 = os.path.basename(image_filename)[:-4]
seg_filename = seg_filenames[i]
f2 = os.path.basename(image_filename)[:-4]
if f1 != f2:
raise SystemError(
'Mismatch between image and label filenames: {}, {}'.
format(f1, f2))
# Read the image.
image_data = tf.gfile.FastGFile(image_filename, 'r').read()
height, width = image_reader.read_image_dims(image_data)
# Read the semantic segmentation annotation.
seg_data = tf.gfile.FastGFile(seg_filename, 'r').read()
seg_height, seg_width = label_reader.read_image_dims(seg_data)
if height != seg_height or width != seg_width:
raise RuntimeError(
'Shape mismatched between image and label.')
# Convert to tf example.
example = build_data.image_seg_to_tfexample(
image_data, image_filename, height, width, seg_data)
tfrecord_writer.write(example.SerializeToString())
sys.stdout.write('\n')
sys.stdout.flush()
def _convert_dataset(dataset_split):
"""Converts the specified dataset split to TFRecord format.
Args:
dataset_split: The dataset split (e.g., train, test)
Raises:
RuntimeError: If loaded image and label have different shape.
"""
# get img filenames.
img_pattern = os.path.join(FLAGS.image_dir, dataset_split, '*',
'*' + IMG_POSTFIX)
img_list = sorted(glob.glob(img_pattern))
# get annotation filenames.
ann_postfix = ANN_POSTFIX_DICT[os.path.basename(FLAGS.annotation_dir)]
ann_pattern = os.path.join(FLAGS.annotation_dir, dataset_split, '*',
'*' + ann_postfix)
ann_list = sorted(glob.glob(ann_pattern))
image_reader = build_data.ImageReader(channels=3)
label_reader = build_data.ImageReader(channels=1)
if not os.path.exists(FLAGS.output_dir):
os.makedirs(FLAGS.output_dir)
assert len(img_list) == len(ann_list)
num_img = len(img_list)
num_img_per_shard = math.ceil(num_img / NUM_SHARD)
print('The number of %s image: %d' % (dataset_split, num_img))
split_identifier = dataset_split + '_' + FILENAME_IDENTIFIER[
os.path.basename(FLAGS.annotation_dir)]
for shard_id in range(NUM_SHARD):
output_filename = os.path.join(
FLAGS.output_dir, '%s-%05d-of%05d.tfrecord' %
(split_identifier, shard_id + 1, NUM_SHARD))
start_idx = shard_id * num_img_per_shard
end_idx = min((shard_id + 1) * num_img_per_shard, num_img)
with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer:
for i in range(start_idx, end_idx):
img_path = img_list[i]
ann_path = ann_list[i]
sys.stdout.write('\r>> Converting image %d/%d shard %d' %
(i + 1, num_img, shard_id))
sys.stdout.flush()
assert (os.path.basename(img_path).replace(
IMG_POSTFIX,
'') == os.path.basename(ann_path).replace(ann_postfix, ''))
# Read the image
img_data = tf.gfile.FastGFile(img_path, 'rb').read()
height, width = image_reader.read_image_dims(img_data)
ann_data = tf.gfile.FastGFile(ann_path, 'rb').read()
ann_height, ann_width = label_reader.read_image_dims(ann_data)
if height != ann_height or width != ann_width:
raise RuntimeError(
'Shape mismatched between image and annotation.')
# Convert to tf example.
example = build_data.image_seg_to_tfexample(
os.path.basename(img_path), img_data, height, width,
ann_data)
tfrecord_writer.write(example.SerializeToString())
sys.stdout.write('\n')
sys.stdout.flush()
def _convert_dataset(dataset_split):
"""Converts the specified dataset split to TFRecord format.
Args:
dataset_split: The dataset split (e.g., train, test).
Raises:
RuntimeError: If loaded image and label have different shape.
"""
# get the image and annotation path from dataset text.
dataset = os.path.basename(dataset_split)[:-4]
sys.stdout.write('Processing ' + dataset + '\n')
img_list = []
ann_list = []
img_dir = FLAGS.image_dir
ann_dir = FLAGS.annotation_dir
with open(dataset_split, 'r') as f:
for row in f:
row = row.strip()
img_path = os.path.join(img_dir, row) + '.jpg'
ann_path = os.path.join(ann_dir, row) + '.png'
img_list.append(img_path)
ann_list.append(ann_path)
image_reader = build_data.ImageReader(channels=3, img_format='jpeg')
label_reader = build_data.ImageReader(channels=1)
if not os.path.exists(FLAGS.output_dir):
os.makedirs(FLAGS.output_dir)
assert len(img_list) == len(ann_list)
# create each video type tfrecord
num_img = len(img_list)
num_img_per_shard = math.ceil(num_img / NUM_SHARD)
print('The number of %s image: %d' % (dataset_split, num_img))
for shard_id in range(NUM_SHARD):
output_filename = os.path.join(
FLAGS.output_dir,
'%s-%05d-of-%05d.tfrecord' % (dataset, shard_id + 1, NUM_SHARD))
start_idx = shard_id * num_img_per_shard
end_idx = min((shard_id + 1) * num_img_per_shard, num_img)
with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer:
for i in range(start_idx, end_idx):
img_path = img_list[i]
ann_path = ann_list[i]
sys.stdout.write('\r>> Converting image %d/%d shard %d' %
(i + 1, num_img, shard_id))
sys.stdout.flush()
# Read the image
img_data = tf.gfile.GFile(img_path, 'rb').read()
height, width = image_reader.read_image_dims(img_data)
ann_data = tf.gfile.GFile(ann_path, 'rb').read()
ann_height, ann_width = label_reader.read_image_dims(ann_data)
if height != ann_height or width != ann_width:
raise RuntimeError(
'Shape mismatched between image and annotations.')
# Convert to tf example.
example = build_data.image_seg_to_tfexample(
os.path.basename(img_path), img_data, height, width,
ann_data)
tfrecord_writer.write(example.SerializeToString())
sys.stdout.write('\n')
sys.stdout.flush()
def _convert_dataset(dataset_split, _NUM_SHARDS, structure_path, plane):
"""Converts the specified dataset split to TFRecord format.
Args:
dataset_split: The dataset split (e.g., train, test).
Raises:
RuntimeError: If loaded image and label have different shape.
"""
image_folder = os.path.join(structure_path, 'processed', 'PNGImages')
semantic_segmentation_folder = os.path.join(structure_path, 'processed',
'SegmentationClass')
image_format = label_format = 'png'
if not os.path.exists(
os.path.join(structure_path, 'tfrecord' + '_' + plane)):
os.makedirs(os.path.join(structure_path, 'tfrecord' + '_' + plane))
dataset = os.path.basename(dataset_split)[:-4]
sys.stdout.write('Processing ' + dataset)
filenames = [x.strip('\n') for x in open(dataset_split, 'r')]
filenames.sort()
# random.shuffle(filenames)
print(filenames)
num_images = len(filenames)
num_per_shard = int(math.ceil(num_images / float(_NUM_SHARDS)))
image_reader = build_data.ImageReader('png', channels=3)
label_reader = build_data.ImageReader('png', channels=1)
for shard_id in range(_NUM_SHARDS):
output_filename = os.path.join(
structure_path, 'tfrecord' + '_' + plane,
'%s-%05d-of-%05d.tfrecord' % (dataset, shard_id, _NUM_SHARDS))
with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer:
start_idx = shard_id * num_per_shard
end_idx = min((shard_id + 1) * num_per_shard, num_images)
for i in range(start_idx, end_idx):
sys.stdout.write('\r>> Converting image %d/%d shard %d' %
(i + 1, len(filenames), shard_id))
sys.stdout.flush()
# Read the image.
image_filename = os.path.join(
image_folder, filenames[i] + '.' + image_format)
image_data = tf.gfile.FastGFile(image_filename, 'rb').read()
height, width = image_reader.read_image_dims(image_data)
# Read the semantic segmentation annotation.
seg_filename = os.path.join(semantic_segmentation_folder,
filenames[i] + '.' + label_format)
seg_data = tf.gfile.FastGFile(seg_filename, 'rb').read()
seg_height, seg_width = label_reader.read_image_dims(seg_data)
if height != seg_height or width != seg_width:
raise RuntimeError(
'Shape mismatched between image and label.')
# Convert to tf example.
example = build_data.image_seg_to_tfexample(
image_data, str.encode(filenames[i], 'utf-8'), height,
width, seg_data)
tfrecord_writer.write(example.SerializeToString())
sys.stdout.write('\n')
sys.stdout.flush()
def _convert_dataset(dataset_split):
"""Converts the specified dataset split to TFRecord format.
Args:
dataset_split: The dataset split (e.g., train, val, test).
Raises:
RuntimeError: If loaded image and label have different shape.
"""
if not os.path.isdir(FLAGS.output_dir):
os.makedirs(FLAGS.output_dir)
dataset = os.path.basename(dataset_split)[:-4]
sys.stdout.write('Processing ' + dataset)
filenames = [x.strip('\n').split(',') for x in open(dataset_split, 'r')]
num_images = len(filenames)
num_per_shard = int(math.ceil(num_images / float(_NUM_SHARDS)))
image_reader = build_data.ImageReader('png', channels=3)
label_reader = build_data.ImageReader('png', channels=1)
# print("n images:", num_images)
# print("n per shard:", num_per_shard)
# print(filenames[0])
for shard_id in range(_NUM_SHARDS):
output_filename = os.path.join(
FLAGS.output_dir,
'%s-%05d-of-%05d.tfrecord' % (dataset, shard_id, _NUM_SHARDS))
if os.path.isfile(output_filename):
continue
with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer:
start_idx = shard_id * num_per_shard
end_idx = min((shard_id + 1) * num_per_shard, num_images)
for i in range(start_idx, end_idx):
sys.stdout.write('\r>> Converting image %d/%d shard %d' %
(i + 1, len(filenames), shard_id))
sys.stdout.flush()
# Read the image data
image_filename = os.path.join(FLAGS.image_folder,
filenames[i][0])
with tf.gfile.FastGFile(image_filename, 'rb') as f:
image_data = f.read()
#image_data = tf.gfile.FastGFile(image_filename, 'rb').read()
## Resize image
image = image_reader.decode_image(image_data)
image = resize_image(image)
image = tf.image.resize_images(
image,
size=[400, 624],
method=tf.image.ResizeMethod.BILINEAR,
align_corners=False)
with tf.Session().as_default():
img = image.eval()
cv2.imwrite(
"check_localization/test_images/tfrecord-test-image-{}.png"
.format(i), img)
height, width = image.shape[:2]
image_data = image_reader.encode_image(image)
# height, width = image_reader.read_image_dims(image_data)
# Read the semantic segmentation annotation.
seg_filename = os.path.join(FLAGS.semantic_segmentation_folder,
filenames[i][1])
with tf.gfile.FastGFile(seg_filename, 'rb') as f:
seg_data = f.read()
#seg_data = tf.gfile.FastGFile(seg_filename, 'rb').read()
## Resize mask
seg_image = label_reader.decode_image(seg_data)
seg_image = resize_image(seg_image)
seg_image = tf.image.resize_images(
seg_image,
size=[400, 624],
method=tf.image.ResizeMethod.BILINEAR,
align_corners=False)
with tf.Session().as_default():
mask = seg_image.eval()
cv2.imwrite(
"check_localization/test_images/tfrecord-test-annot-{}.png"
.format(i), mask)
seg_height, seg_width = seg_image.shape[:2]
seg_data = label_reader.encode_image(seg_image)
# seg_height, seg_width = label_reader.read_image_dims(seg_data, resize=True)
if height != seg_height or width != seg_width:
raise RuntimeError(
'Shape mismatched between image and label.')
# Convert to tf example.
example = build_data.image_seg_to_tfexample(
image_data, filenames[i][0], height, width, seg_data)
tfrecord_writer.write(example.SerializeToString())
sys.stdout.write('\n')
sys.stdout.flush()
del example, tfrecord_writer,
break
def aug_write_tfexample(image_name, image_reader, label_reader, train_df,
tfrecord_writer):
# Read the image.
image_filename = os.path.join(FLAGS.train_folder, image_name)
image_data = tf.gfile.FastGFile(image_filename, 'rb').read()
height, width = image_reader.read_image_dims(image_data)
# Read the semantic segmentation annotation.
mask_list = train_df['EncodedPixels'][train_df['ImageId'] ==
image_name].tolist()
seg_mask = np.zeros((768, 768, 1))
for item in mask_list:
rle_list = str(item).split()
tmp_mask = rle_to_mask(rle_list, (768, 768))
seg_mask[:, :, 0] += tmp_mask
seg_filename = os.path.join(FLAGS.seg_folder, image_name[:-4] + '.png')
cv2.imwrite(seg_filename, seg_mask)
seg_data = tf.gfile.FastGFile(seg_filename, 'rb').read()
seg_height, seg_width = label_reader.read_image_dims(seg_data)
if height != seg_height or width != seg_width:
raise RuntimeError('Shape mismatched between image and label.')
img = cv2.imread(image_filename)
M = cv2.getRotationMatrix2D((384, 384), 180, 1.0)
N = cv2.getRotationMatrix2D((384, 384), 90, 1.0)
rotate_imga = cv2.warpAffine(img, M, (768, 768))
rotate_sega = cv2.warpAffine(seg_mask, M, (768, 768))
rotate_imgb = cv2.warpAffine(img, N, (768, 768))
rotate_segb = cv2.warpAffine(seg_mask, N, (768, 768))
flip_img = cv2.flip(img, 0)
flip_seg = cv2.flip(seg_mask, 0)
imga_filename = os.path.join(FLAGS.train_folder,
image_name[:-4] + '_a.jpg')
imgb_filename = os.path.join(FLAGS.train_folder,
image_name[:-4] + '_b.jpg')
imgf_filename = os.path.join(FLAGS.train_folder,
image_name[:-4] + '_f.jpg')
cv2.imwrite(imga_filename, rotate_imga)
cv2.imwrite(imgb_filename, rotate_imgb)
cv2.imwrite(imgf_filename, flip_img)
imga_data = tf.gfile.FastGFile(imga_filename, 'rb').read()
imgb_data = tf.gfile.FastGFile(imgb_filename, 'rb').read()
imgf_data = tf.gfile.FastGFile(imgf_filename, 'rb').read()
sega_filename = os.path.join(FLAGS.seg_folder, image_name[:-4] + '_a.png')
segb_filename = os.path.join(FLAGS.seg_folder, image_name[:-4] + '_b.png')
segf_filename = os.path.join(FLAGS.seg_folder, image_name[:-4] + '_f.png')
cv2.imwrite(sega_filename, rotate_sega)
cv2.imwrite(segb_filename, rotate_segb)
cv2.imwrite(segf_filename, flip_seg)
sega_data = tf.gfile.FastGFile(sega_filename, 'rb').read()
segb_data = tf.gfile.FastGFile(segb_filename, 'rb').read()
segf_data = tf.gfile.FastGFile(segf_filename, 'rb').read()
# Convert to tf example.
example = build_data.image_seg_to_tfexample(image_data, image_name[:-4],
height, width, seg_data)
tfrecord_writer.write(example.SerializeToString())
example_a = build_data.image_seg_to_tfexample(imga_data,
image_name[:-4] + '_a',
height, width, sega_data)
tfrecord_writer.write(example_a.SerializeToString())
example_b = build_data.image_seg_to_tfexample(imgb_data,
image_name[:-4] + '_b',
height, width, segb_data)
tfrecord_writer.write(example_b.SerializeToString())
example_f = build_data.image_seg_to_tfexample(imgf_data,
image_name[:-4] + '_f',
height, width, segf_data)
tfrecord_writer.write(example_f.SerializeToString())
def _convert_dataset(dataset_split,
dataset_name='',
class_ignore_value=255,
num_shards=1,
num_imgs=-1,
remaining_imgs_type=None,
remaining_imgs_num_or_ratio=None,
overwrite=False,
verbose=True,
shuffle=False,
shuffle_seed=1234):
"""Converts the specified dataset split to TFRecord format.
Args:
dataset_split: The dataset split (e.g., train, val).
dataset_name: The dataset name (e.g., train_hints, val_hints). Default is set to dataset_split. This is the name of the tfrecord.
remaining_imgs_type: if num_imgs is set, what should we do we remaining images if remaining_imgs_num_or_ratio is set?
None: Use remaining image labels as-is
path -- string: Use labels from path (use alternative labels)
remaining_imgs_num_or_ratio::
None: don't use remaining images (same as 0 or 0.0)
ratio -- float between 0 and 1: use ratio * num_remaining_imgs remaining images.
num -- integer: use num remaining images
Raises:
RuntimeError: If loaded image and label have different shape, or if the
image file with specified postfix could not be found.
"""
sys.stdout.flush()
print('###############################################')
sys.stdout.write('\rWorking on: {}\n'.format(dataset_name))
image_files = []
label_files = []
print('Using full res images and labels...')
image_files.extend(_get_files('image', dataset_split))
if 'coarse' in dataset_name:
label_files.extend(_get_files('label_coarse', dataset_split))
elif 'block' in dataset_name:
label_files.extend(_get_files('label_block', dataset_split))
else:
label_files.extend(_get_files('label', dataset_split))
if num_imgs < 0:
num_images = len(image_files)
else:
num_images = num_imgs
remaining_imgs_label_files = None
num_remaining_imgs_num = None
if remaining_imgs_num_or_ratio is not None:
if type(remaining_imgs_num_or_ratio) == float:
assert 0 <= remaining_imgs_num_or_ratio <= 1
num_remaining_images = int(remaining_imgs_num_or_ratio *
(len(image_files) - num_images))
if type(remaining_imgs_num_or_ratio) == int:
assert 0 <= remaining_imgs_num_or_ratio
num_remaining_images = min(remaining_imgs_num_or_ratio,
(len(image_files) - num_images))
if remaining_imgs_type is None:
remaining_imgs_label_files = list(label_files)
elif type(remaining_imgs_type) == str:
print("Searching {} for label files.")
remaining_imgs_label_files = []
for root, dirnames, filenames in os.walk(remaining_imgs_type):
for filename in fnmatch.filter(filenames, "*"):
remaining_imgs_label_files.append(
os.path.join(root, filename))
remaining_imgs_label_files = sorted(remaining_imgs_label_files)
assert len(remaining_imgs_label_files) == len(
label_files
), 'Expected {} alternative label files; found {}'.format(
len(label_files), len(remaining_imgs_label_files))
else:
raise TypeError("remaining_imgs_type should be a string or None")
if shuffle:
shuffled_idxs = np.arange(len(image_files))
np.random.seed(shuffle_seed)
np.random.shuffle(shuffled_idxs)
print('Using indices {} ...'.format(shuffled_idxs[:10]))
# Image, label, boundary distance map
image_files = np.array(image_files)[shuffled_idxs]
label_files = np.array(label_files)[shuffled_idxs]
# Alternative label files
if remaining_imgs_label_files is not None:
remaining_imgs_label_files = np.array(
remaining_imgs_label_files)[shuffled_idxs]
# Concat num_images label_files with num_remaining_images_label_files
label_files = list(label_files)[:num_images] + list(
remaining_imgs_label_files)[num_images:num_images +
num_remaining_images]
assert len(label_files) == num_images + num_remaining_images, (
len(label_files), num_images, num_remaining_images)
num_images = num_images + num_remaining_images
if not shuffle and remaining_imgs_label_files is not None:
raise NotImplementedError("This is not going to work; check the code")
num_per_shard = int(math.ceil(num_images / float(num_shards)))
image_reader = build_data.ImageReader('png', channels=3)
label_reader = build_data.ImageReader('png', channels=1)
for shard_id in range(num_shards):
if dataset_name == '':
dataset_name = dataset_split
shard_filename = '%s-%05d-of-%05d.tfrecord' % (dataset_name, shard_id,
num_shards)
output_filename = os.path.join(FLAGS.output_dir, shard_filename)
if os.path.exists(output_filename) and not overwrite:
print('File exists. Skipping. {}'.format(output_filename))
continue
with tf.python_io.TFRecordWriter(output_filename) as tfrecord_writer:
start_idx = shard_id * num_per_shard
end_idx = min((shard_id + 1) * num_per_shard, num_images)
for i in range(start_idx, end_idx):
sys.stdout.flush()
sys.stdout.write('\r>> Converting image %d/%d shard %d' %
(i + 1, num_images, shard_id))
#sys.stdout.flush()
# Read the image.
image_data = tf.gfile.FastGFile(image_files[i], 'rb').read()
height, width = image_reader.read_image_dims(image_data)
# Read the semantic segmentation annotation.
seg_data = tf.gfile.FastGFile(label_files[i], 'rb').read()
seg_height, seg_width = label_reader.read_image_dims(seg_data)
if verbose:
sys.stdout.write('\r\nUsing\n {}\n {}\n'.format(
image_files[i], label_files[i]))
if height != seg_height or width != seg_width:
raise RuntimeError(
'Shape mismatched between image and label.')
# Convert to tf example.
base_filename = os.path.basename(image_files[i]).\
replace(_POSTFIX_MAP['image']+'.'+_DATA_FORMAT_MAP['image'], '')
filename = base_filename
example = build_data.image_seg_to_tfexample(
image_data=image_data,
filename=filename,
height=height,
width=width,
seg_data=seg_data,
)
tfrecord_writer.write(example.SerializeToString())
sys.stdout.write('\n')
sys.stdout.flush()
