def prepare(tasks, max_images, data_dir, redownload):
if "ffhq" in tasks:
task("ffhq", "FFHQ", 13, data_dir, redownload,
download = lambda: download_drive_files(data_dir, urls["ffhq"]))
if "clevr" in tasks:
task("clevr", "CLEVR", 18, data_dir, redownload,
download = lambda: download_file(urls["clevr"], data_dir, unzip = True),
prepare = lambda: create_from_imgs("{}/clevr".format(data_dir), "{}/TODO".format(data_dir),
ratio = 0.75))
if "bedrooms" in tasks:
task("bedrooms", "LSUN-Bedrooms", 43, data_dir, redownload,
download = lambda: download_file(urls["clevr"], data_dir, unzip = True),
prepare = lambda: create_from_tfds("{}/bedrooms".format(data_dir), "lsun/bedroom",
ratio = 188/256))
"img_num": 100000
##########################################################################################
# Currently, we download preprocessed TFrecords of CLEVR images with image ratio 0.75.
# To process instead the dataset from scratch (with the original image ratio of 320/480), add the following:
# "filename": "CLEVR_v1.0.zip",
# "size": 18,
# "dir": "CLEVR_v1.0/images", # Image directory to process while turning into tfrecords
# "url": "https://dl.fbaipublicfiles.com/clevr/CLEVR_v1.0.zip",
# "md5": "b11922020e72d0cd9154779b2d3d07d2",
# "process": dataset_tool.create_from_imgs # Function to convert download to tfrecords
})
}
formats_catalog = {
"png":
lambda tfdir, imgdir, **kwargs: dataset_tool.create_from_imgs(
tfdir, imgdir, format="png", **kwargs),
"jpg":
lambda tfdir, imgdir, **kwargs: dataset_tool.create_from_imgs(
tfdir, imgdir, format="jpg", **kwargs),
"npy":
dataset_tool.create_from_npy,
"hdf5":
dataset_tool.create_from_hdf5,
"tfds":
dataset_tool.create_from_tfds,
"lmdb":
dataset_tool.create_from_lmdb
}
def mkdir(d):
"img_num": 100000,
"process": dataset_tool.create_from_tfrecords # Function to convert download to tfrecords
##########################################################################################
# Currently, we download preprocessed TFrecords of CLEVR images with image ratio 0.75.
# To process instead the dataset from scratch (with the original image ratio of 320/480), add the following:
# "filename": "CLEVR_v1.0.zip",
# "size": 18,
# "dir": "CLEVR_v1.0/images", # Image directory to process while turning into tfrecords
# "url": "https://dl.fbaipublicfiles.com/clevr/CLEVR_v1.0.zip",
# "md5": "b11922020e72d0cd9154779b2d3d07d2",
# "process": dataset_tool.create_from_imgs # Function to convert download to tfrecords
})
}
formats_catalog = {
"png": lambda datadir, imgdir, **kwargs: dataset_tool.create_from_imgs(datadir, imgdir, format = "png", **kwargs),
"jpg": lambda datadir, imgdir, **kwargs: dataset_tool.create_from_imgs(datadir, imgdir, format = "jpg", **kwargs),
"npy": dataset_tool.create_from_npy,
"hdf5": dataset_tool.create_from_hdf5,
"tfds": dataset_tool.create_from_tfds,
"lmdb": dataset_tool.create_from_lmdb,
"tfrecords": dataset_tool.create_from_tfrecords
}
def verify_md5(filename, md5):
print(f"Verify MD5 for {filename}...")
with open(filename, "rb") as f:
new_md5 = hashlib.md5(f.read()).hexdigest()
result = md5 == new_md5
if result:
print(misc.bold("MD5 matches!"))