def make_dataset(classlist, labellist=None):
images = []
labels = []
classes = utils.readtextfile(ifile)
classes = [x.rstrip('\n') for x in classes]
classes.sort()
for i in len(classes):
for fname in os.listdir(classes[i]):
if is_image_file(fname):
label = {}
label['class'] = os.path.split(classes[i])
images.append(fname)
labels.append(label)
if labellist != None:
labels = utils.readtextfile(ifile)
labels = [x.rstrip('\n') for x in labels]
labels.sort()
for i in len(labels):
for fname in os.listdir(labels[i]):
if is_image_file(fname):
labels.append(os.path.split(classes[i]))
return images, labels
def make_dataset(classlist):
filename = '/tmp/folderlist.pkl'
if utils.file_exists(filename):
print("loading from cache")
pickle_load = pickle.load(open(filename, "rb"))
images = pickle_load["images"]
labels = pickle_load["labels"]
else:
print("cache not found, generating cache, this will take a while")
images = []
labels = []
classes = utils.readtextfile(classlist)
classes = [x.rstrip('\n') for x in classes]
classes.sort()
for index in range(len(classes)):
for fname in os.listdir(classes[index]):
if is_image_file(fname):
fname = os.path.join(classes[index], fname)
images.append(fname)
labels.append(index)
pickle_save = {"images": images, "labels": labels}
pickle.dump(pickle_save, open(filename, "wb"))
return images, labels
def __init__(self,
ifile,
root=None,
split=1.0,
transform=None,
loader='loader_image'):
self.root = root
self.transform = transform
if loader is not None:
self.loader = getattr(loaders, loader)
self.f_h5py = h5py.File(ifile[0], 'r')
if ifile[1].endswith('txt'):
lines = utils.readtextfile(ifile[1])
imagelist = []
for x in lines:
x = x.rstrip('\n')
filename = os.path.splitext(os.path.basename(x))[0]
labelname = os.path.basename(os.path.dirname(x))
temp = [os.path.join(labelname, filename + '.jpg')]
temp.append(labelname)
imagelist.append(temp)
labellist = [x[1] for x in imagelist]
self.images = imagelist
self.classname = labellist
self.classname = list(set(self.classname))
self.classname.sort()
def __init__(self,
ifile,
root=None,
split=1.0,
transform=None,
loader='loader_image'):
self.ifile = ifile
self.root = root
self.split = split
self.transform = transform
if loader is not None:
self.loader = getattr(loaders, loader)
if ifile is not None:
if ifile.endswith('txt'):
lines = utils.readtextfile(ifile)
imagelist = []
for x in lines:
x = x.rstrip('\n')
filename = os.path.splitext(os.path.basename(x))[0]
file_ext = os.path.splitext(os.path.basename(x))[1]
labelname = os.path.basename(os.path.dirname(x))
# temp = [os.path.join(labelname, filename + file_ext)]
temp = [x]
temp.append(labelname)
imagelist.append(temp)
labellist = [x[1] for x in imagelist]
else:
imagelist = []
if (self.split < 1.0) & (self.split > 0.0):
if len(imagelist) > 0:
imagelist = shuffle(imagelist, labellist)
num = math.floor(self.split * len(imagelist))
self.images = imagelist[0:num]
else:
self.images = []
elif self.split == 1.0:
if len(imagelist) > 0:
self.images = imagelist
else:
self.images = []
if len(labellist) > 0:
self.classname = labellist
else:
self.classname = []
self.classname = list(set(self.classname))
self.classname.sort()
def __init__(self,
trainfile,
testfile,
split_train=1.0,
split_test=0.0,
train=True,
transform_train=None,
transform_test=None,
loader_train=loaders.loader_numpy,
loader_test=loaders.loader_numpy):
# TODO: Split_train is not currently used. Either make use of it or
# delete it.
self.trainfile = trainfile
self.testfile = testfile
self.train = train
self.split_test = split_test
self.split_train = split_train
self.transform_test = transform_test
self.transform_train = transform_train
self.loader_train = loader_train
self.loader_test = loader_test
self.preload = False
if loader_train == 'image':
self.loader_train = loaders.loader_image
if loader_train == 'torch':
self.loader_train = loaders.loader_torch
if loader_train == 'numpy':
self.loader_train = loaders.loader_numpy
if loader_train == 'h5py':
self.loader_train = loaders.loader_h5py
if loader_test == 'image':
self.loader_test = loaders.loader_image
if loader_test == 'torch':
self.loader_test = loaders.loader_torch
if loader_test == 'numpy':
self.loader_test = loaders.loader_numpy
if loader_test == 'h5py':
self.loader_test = loaders.loader_h5py
if trainfile is not None:
trainlist = utils.readtextfile(trainfile)
trainlist = [x.rstrip('\n') for x in trainlist]
else:
trainlist = []
if testfile is not None:
testlist = utils.readtextfile(testfile)
testlist = [x.rstrip('\n') for x in testlist]
else:
testlist = []
if len(trainlist) == len(testlist):
shuffle(testlist, trainlist)
if len(trainlist) > 0 and len(testlist) == 0:
shuffle(trainlist)
if len(testlist) > 0 and len(trainlist) == 0:
shuffle(testlist)
if (self.split_train < 1.0) & (self.split_test > 0.0):
if len(trainlist) > 0:
num = math.floor(self.split_train * len(trainlist))
self.image_train = trainlist[0:num]
if len(testlist) > 0:
num = math.floor(self.split_train * len(testlist))
self.image_test = testlist[num + 1:len(testlist)]
elif self.split_train == 1.0:
if len(trainlist) > 0:
self.image_train = trainlist
else:
raise NotImplementedError("Data not found")
elif self.split_test == 1.0:
if len(testlist) > 0:
self.image_test = testlist
if self.preload is True:
self.image_train_data = []
for i in self.image_train:
self.image_train_data.append(self.loader_train(i))
def __init__(self, filler_images, block_sizes=[25, 35, 45, 55, 65, 75, 85, 95, 105, 115]):
self.block_sizes = block_sizes
self.toTensor = torchvision.transforms.ToTensor()
self.filler_image_list = [x.rstrip('\n') for x in utils.readtextfile(filler_images)]
assert (len(self.filler_image_list) > 0)
def __init__(self,
ifile,
lfile=None,
split_train=1.0,
split_test=0.0,
train=True,
transform_train=None,
transform_test=None,
loader_input='image',
loader_label='torch'):
self.ifile = ifile
self.lfile = lfile
self.train = train
self.split_test = split_test
self.split_train = split_train
self.transform_test = transform_test
self.transform_train = transform_train
if loader_input == 'image':
self.loader_input = loaders.loader_image
if loader_input == 'torch':
self.loader_input = loaders.loader_torch
if loader_input == 'numpy':
self.loader_input = loaders.loader_numpy
if loader_label == 'image':
self.loader_label = loaders.loader_image
if loader_label == 'torch':
self.loader_label = loaders.loader_torch
if loader_label == 'numpy':
self.loader_label = loaders.loader_numpy
if ifile is not None:
imagelist = utils.readtextfile(ifile)
imagelist = [x.rstrip('\n') for x in imagelist]
else:
imagelist = []
if lfile is not None:
labellist = utils.readtextfile(lfile)
labellist = [x.rstrip('\n') for x in labellist]
else:
labellist = []
if len(imagelist) == len(labellist):
shuffle(imagelist, labellist)
if len(imagelist) > 0 and len(labellist) == 0:
shuffle(imagelist)
if len(labellist) > 0 and len(imagelist) == 0:
shuffle(labellist)
if (self.split_train < 1.0) & (self.split_train > 0.0):
if len(imagelist) > 0:
num = math.floor(self.split_train * len(imagelist))
self.images_train = imagelist[0:num]
self.images_test = imagelist[num + 1:len(imagelist)]
else:
self.images_test = []
self.images_train = []
if len(labellist) > 0:
num = math.floor(self.split * len(labellist))
self.labels_train = labellist[0:num]
self.labels_test = labellist[num + 1:len(labellist)]
else:
self.labels_test = []
self.labels_train = []
elif self.split_train == 1.0:
if len(imagelist) > 0:
self.images_train = imagelist
else:
self.images_train = []
if len(labellist) > 0:
self.labels_train = labellist
else:
self.labels_train = []
elif self.split_test == 1.0:
if len(imagelist) > 0:
self.images_test = imagelist
else:
self.images_test = []
if len(labellist) > 0:
self.labels_test = labellist
else:
self.labels_test = []