def _generate_data():
"""Generate images archive."""
# Generate images
images_dir = os.path.join(_output_dir(), 'images')
if not tf.io.gfile.exists(images_dir):
tf.io.gfile.makedirs(images_dir)
for i in range(_TRAIN_IMAGES_NUMBER + _TEST_IMAGES_NUMBER):
image_name = 'image{:03d}.jpg'.format(i)
tf.io.gfile.copy(fake_data_utils.get_random_jpeg(),
os.path.join(images_dir, image_name),
overwrite=True)
# Generate annotations
annotations_dir = os.path.join(_output_dir(), 'annotations')
if not tf.io.gfile.exists(annotations_dir):
tf.io.gfile.makedirs(annotations_dir)
global_count = 0
for filename, num_examples in [('trainval.txt', _TRAIN_IMAGES_NUMBER),
('test.txt', _TEST_IMAGES_NUMBER)]:
fobj = tempfile.NamedTemporaryFile(delete=False, mode='w')
with fobj:
for i in range(num_examples):
fobj.write('image{:03d} {} 0 0\n'.format(global_count, i % 37))
global_count += 1
tf.io.gfile.copy(fobj.name,
os.path.join(annotations_dir, filename),
overwrite=True)
def _generate_image(data, fdir, fname):
dirname = os.path.join(_output_dir(data), fdir)
if not os.path.exists(dirname):
os.makedirs(dirname)
tf.io.gfile.copy(fake_data_utils.get_random_jpeg(480, 720),
os.path.join(dirname, fname),
overwrite=True)
def main(argv):
del argv
out_path = os.path.join(_output_dir(), 'cats_vs_dogs.zip')
jpg = fake_data_utils.get_random_jpeg(height=1, width=1)
with zipfile.ZipFile(out_path, 'w') as myzip:
myzip.write(jpg, 'PetImages/Dog/0.jpg')
myzip.write(jpg, 'PetImages/Dog/1.jpg')
myzip.write(jpg, 'PetImages/Cat/0.jpg')
myzip.write(jpg, 'PetImages/Cat/1.jpg')
def _generate_jpeg(example_id, height, width):
"""Generate a fake jpeg image for the given example id."""
jpeg = fake_data_utils.get_random_jpeg(height=height, width=width)
filepath = os.path.join(_voc2007_output_dir(),
"VOCdevkit/VOC2007/JPEGImages/%06d.jpg" % example_id)
dirname = os.path.dirname(filepath)
if not tf.io.gfile.exists(dirname):
tf.io.gfile.makedirs(dirname)
tf.io.gfile.copy(jpeg, filepath, overwrite=True)
def _generate_val_archive():
"""Generate val archive."""
output_path = os.path.join(_ilsvrc2012_output_dir(),
'ILSVRC2012_img_val.tar')
tar = tarfile.open(output_path, mode='w')
for i in range(1, VAL_IMAGES_NUMBER + 1):
fname = 'ILSVRC2012_val_0000%03i.JPEG' % i
jpeg = fake_data_utils.get_random_jpeg()
tar.add(jpeg, arcname=fname)
tar.close()
def _get_synset(synset_name):
"""Returns path to synset archive."""
fobj = tempfile.NamedTemporaryFile(delete=False, mode='wb', suffix='.tar')
tar = tarfile.open(mode='w', fileobj=fobj)
for i in range(1, TRAIN_IMAGES_PER_SYNSET + 1):
fname = '%s_%s.JPEG' % (synset_name, i)
jpeg = fake_data_utils.get_random_jpeg()
tar.add(jpeg, arcname=fname)
fobj.close()
return fobj.name
def _generate_jpeg(example_id, height, width):
"""Generate a fake jpeg image for the given example id."""
jpeg = fake_data_utils.get_random_jpeg(height=height, width=width)
filepath = os.path.join(
_output_dir(),
"BCCD_Dataset-1.0/BCCD/JPEGImages/BloodImage_{:05d}.jpg".format(
example_id))
dirname = os.path.dirname(filepath)
if not tf.io.gfile.exists(dirname):
tf.io.gfile.makedirs(dirname)
tf.io.gfile.copy(jpeg, filepath, overwrite=True)
def _write_tar(path, split_name, image_ids, prefix=None):
"""Writes tar file with images to given path.
Args:
path: sting, path to tar to be written.
split_name: string. eg: 'train', 'validation', 'test'.
image_ids: list of str, ids of the images to add to tar file.
prefix: one of [0-9a-f], or None.
"""
if prefix is not None:
split_name = '%s_%s' % (split_name, prefix)
with tarfile.open(path, mode='w') as tar:
for image_id in image_ids:
fname = '%s/%s.jpg' % (split_name, image_id)
tar.add(fake_data_utils.get_random_jpeg(), arcname=fname)
def _generate_val_archive():
"""Generate val archive."""
output_path = os.path.join(_ilsvrc2012_output_dir(),
'ILSVRC2012_img_val.tar')
tar = tarfile.open(output_path, mode='w')
for i in range(1, VAL_IMAGES_NUMBER+1):
fname = 'ILSVRC2012_val_{:08}.JPEG'.format(i)
jpeg = fake_data_utils.get_random_jpeg()
tar.add(jpeg, arcname=fname)
tar.close()
if FLAGS.real:
real_labels_path = os.path.join(_ilsvrc2012_output_dir(), 'real.json')
with open(real_labels_path, 'w') as f:
json.dump([[i, i + 1] for i in range(VAL_IMAGES_NUMBER)], f)
def _write_tar(path, split_name, image_ids, prefix=None):
"""Writes tar file with images to given path.
Args:
path: sting, path to tar to be written.
split_name: string. eg: 'train', 'validation', 'test'.
image_ids: list of str, ids of the images to add to tar file.
prefix: one of [0-9a-f], or None.
"""
if prefix is not None:
split_name = '%s_%s' % (split_name, prefix)
with tarfile.open(path, mode='w') as tar:
for i, image_id in enumerate(image_ids):
fname = '%s/%s.jpg' % (split_name, image_id)
# Note: Generate two large images with more than 300k pixels.
kwargs = dict(height=600, width=600) if i < 2 else dict()
tar.add(fake_data_utils.get_random_jpeg(**kwargs), arcname=fname)
def _get_synset(synset_name):
"""Returns path to synset archive."""
fobj = tempfile.NamedTemporaryFile(delete=False, mode='wb', suffix='.tar')
tar = tarfile.open(mode='w', fileobj=fobj)
for i in range(1, TRAIN_IMAGES_PER_SYNSET + 1):
fname = '%s_%s.JPEG' % (synset_name, i)
# There are a few PNG and CMYK images:
if synset_name == 'n01440764' and i == 1:
path = fake_data_utils.get_random_png()
elif synset_name == 'n01440764' and i in [2, 3]:
path = os.path.join(FLAGS.tfds_dir, 'testing', 'test_data',
'6pixels_cmyk.jpeg')
else:
path = fake_data_utils.get_random_jpeg()
tar.add(path, arcname=fname)
fobj.close()
return fobj.name
def _generate_data(split_name, num_examples):
"""Generate test data."""
names_file = tfds.core.tfds_path(
os.path.join("image", "dtd_key_attributes.txt"))
label_names = tfds.features.ClassLabel(names_file=names_file).names
# Generate images.
generated_filenames = []
for n in range(num_examples):
label = random.choice(label_names)
fname = "%s/%s_%04d.jpg" % (label, label, n)
tmp_file_path = fake_data_utils.get_random_jpeg()
dst_file_path = os.path.join(_output_dir(), "dtd", "images", fname)
generated_filenames.append(fname)
_makedir_if_not_exists(os.path.dirname(dst_file_path))
tf.io.gfile.copy(tmp_file_path, dst_file_path, overwrite=True)
split_path = os.path.join(_output_dir(), "dtd", "labels", split_name + ".txt")
_makedir_if_not_exists(os.path.dirname(split_path))
with tf.io.gfile.GFile(split_path, "w") as split_file:
for fname in generated_filenames:
split_file.write("%s\n" % fname)