def _prepare_synset_to_human(self):
SIZE = 2655750
URL = "https://heibox.uni-heidelberg.de/f/9f28e956cd304264bb82/?dl=1"
self.human_dict = os.path.join(self.root, "synset_human.txt")
if (not os.path.exists(self.human_dict) or
not os.path.getsize(self.human_dict)==SIZE):
download(URL, self.human_dict)
def _prepare(self):
cachedir = os.environ.get("XDG_CACHE_HOME",
os.path.expanduser("~/.cache"))
self.root = os.path.join(cachedir, "autoencoders/data", self.NAME)
SRC = {
"AwA2-data.zip": "http://cvml.ist.ac.at/AwA2/AwA2-data.zip",
}
if not edu.is_prepared(self.root):
# prep
self.logger.info("Preparing dataset {} in {}".format(
self.NAME, self.root))
os.makedirs(self.root, exist_ok=True)
datadir = os.path.join(self.root, "Animals_with_Attributes2")
if not os.path.exists(datadir):
datapath = os.path.join(self.root, "AwA2-data.zip")
if not os.path.exists(datapath):
download(SRC["AwA2-data.zip"], datapath)
edu.unpack(datapath)
# make filelist
images = list()
for path, subdirs, files in os.walk(
os.path.join(datadir, "JPEGImages")):
for name in files:
searchname = name.lower()
if (searchname.rfind('jpg') != -1
or searchname.rfind('png') != -1
or searchname.rfind('jpeg') != -1):
filename = os.path.relpath(os.path.join(path, name),
start=self.root)
images.append(filename)
prng = np.random.RandomState(1)
test = set(prng.choice(len(images), 5000, replace=False))
train_images = [
images[i] for i in range(len(images)) if not i in test
]
test_images = [images[i] for i in range(len(images)) if i in test]
with open(os.path.join(self.root, "train.txt"), "w") as f:
f.write("\n".join(train_images) + "\n")
with open(os.path.join(self.root, "test.txt"), "w") as f:
f.write("\n".join(test_images) + "\n")
with open(
os.path.join(self.root,
"Animals_with_Attributes2/classes.txt"),
"r") as f:
classes = f.read().splitlines()
classes = [cls.split()[-1] for cls in classes]
classes = sorted(classes)
with open(os.path.join(self.root, "classes.txt"), "w") as f:
f.write("\n".join(classes) + "\n")
edu.mark_prepared(self.root)
def _prepare_animal_synsets(self):
SIZE = 1490
URL = "https://heibox.uni-heidelberg.de/f/c18cdf02ea0b4e758729/?dl=1"
cachedir = os.environ.get("XDG_CACHE_HOME", os.path.expanduser("~/.cache"))
self.root = os.path.join(cachedir, "autoencoders/data/ImageNetAnimals")
self.animal_synsets = os.path.join(self.root, "animal_synsets.txt")
if (not os.path.exists(self.animal_synsets) or
not os.path.getsize(self.animal_synsets)==SIZE):
download(URL, self.animal_synsets)
def _prepare(self):
self.random_crop = retrieve(self.config,
"ImageNetValidation/random_crop",
default=False)
cachedir = os.environ.get("XDG_CACHE_HOME",
os.path.expanduser("~/.cache"))
self.root = os.path.join(cachedir, "autoencoders/data", self.NAME)
self.datadir = os.path.join(self.root, "data")
self.txt_filelist = os.path.join(self.root, "filelist.txt")
self.expected_length = 50000
if not edu.is_prepared(self.root):
# prep
self.logger.info("Preparing dataset {} in {}".format(
self.NAME, self.root))
datadir = self.datadir
if not os.path.exists(datadir):
path = os.path.join(self.root, self.FILES[0])
if not os.path.exists(path) or not os.path.getsize(
path) == self.SIZES[0]:
import academictorrents as at
atpath = at.get(self.AT_HASH, datastore=self.root)
assert atpath == path
self.logger.info("Extracting {} to {}".format(path, datadir))
os.makedirs(datadir, exist_ok=True)
with tarfile.open(path, "r:") as tar:
tar.extractall(path=datadir)
vspath = os.path.join(self.root, self.FILES[1])
if not os.path.exists(vspath) or not os.path.getsize(
vspath) == self.SIZES[1]:
download(self.VS_URL, vspath)
with open(vspath, "r") as f:
synset_dict = f.read().splitlines()
synset_dict = dict(line.split() for line in synset_dict)
self.logger.info("Reorganizing into synset folders")
synsets = np.unique(list(synset_dict.values()))
for s in synsets:
os.makedirs(os.path.join(datadir, s), exist_ok=True)
for k, v in synset_dict.items():
src = os.path.join(datadir, k)
dst = os.path.join(datadir, v)
shutil.move(src, dst)
filelist = glob.glob(os.path.join(datadir, "**", "*.JPEG"))
filelist = [os.path.relpath(p, start=datadir) for p in filelist]
filelist = sorted(filelist)
filelist = "\n".join(filelist) + "\n"
with open(self.txt_filelist, "w") as f:
f.write(filelist)
edu.mark_prepared(self.root)
def _prepare(self):
cachedir = os.environ.get("XDG_CACHE_HOME", os.path.expanduser("~/.cache"))
self.root = os.path.join(cachedir, "autoencoders/data/AnimalFaces")
self.logger.info("Using data located at {}".format(self.root))
os.makedirs(self.root, exist_ok=True)
self.datadir = os.path.join(self.root, "data")
if not edu.is_prepared(self.root):
self.logger.info("Preparing dataset {} in {}".format(self.NAME, self.root))
if not os.path.exists(self.datadir):
os.makedirs(self.datadir, exist_ok=True)
imagenet = ImageNetTrain()
coor_path = os.path.join(self.root, "animalface_coordinates.txt")
if not os.path.exists(coor_path):
download(self.COOR_URL, coor_path)
with open(coor_path, "r") as f:
animalface_coordinates = f.readlines()
for line in tqdm(animalface_coordinates):
ls = line.strip().split(' ')
img_name = os.path.join(imagenet.datadir, ls[0])
img = Image.open(img_name)
img = img.convert('RGB')
x = int(ls[1])
y = int(ls[2])
w = int(ls[3])
h = int(ls[4])
crop = img.crop((x, y, w, h))
out_name = os.path.join(self.datadir,
'%s_%d_%d_%d_%d.jpg' % (ls[0], x, y, w, h))
os.makedirs(os.path.dirname(out_name), exist_ok=True)
crop.save(out_name)
train_path = os.path.join(self.root, "animals_list_train.txt")
if not os.path.exists(train_path):
download(self.TRAIN_URL, train_path)
test_path = os.path.join(self.root, "animals_list_test.txt")
if not os.path.exists(test_path):
download(self.TEST_URL, test_path)
shared_train_path = os.path.join(self.root, "shared_animalfaces_train.txt")
if not os.path.exists(shared_train_path):
download(self.SHARED_TRAIN_URL, shared_train_path)
shared_test_path = os.path.join(self.root, "shared_animalfaces_test.txt")
if not os.path.exists(shared_test_path):
download(self.SHARED_TEST_URL, shared_test_path)
restricted_train_path = os.path.join(self.root, "restricted_animalfaces_train.txt")
if not os.path.exists(restricted_train_path):
download(self.RESTRICTED_TRAIN_URL, restricted_train_path)
restricted_test_path = os.path.join(self.root, "restricted_animalfaces_test.txt")
if not os.path.exists(restricted_test_path):
download(self.RESTRICTED_TEST_URL, restricted_test_path)
edu.mark_prepared(self.root)
def _prepare_idx_to_synset(self):
URL = "https://heibox.uni-heidelberg.de/f/d835d5b6ceda4d3aa910/?dl=1"
self.idx2syn = os.path.join(self.root, "index_synset.yaml")
if (not os.path.exists(self.idx2syn)):
download(URL, self.idx2syn)