def __init__(self, train_images_dir, val_images_dir, test_images_dir,
train_batch_size, val_batch_size, test_batch_size,
height_of_image, width_of_image, num_channels, num_classes,
model):
self.config = {
"m_train": 55000,
"m_val": 5000,
"m_test": 10000,
"train_batch_size": train_batch_size,
"val_batch_size": val_batch_size,
"test_batch_size": test_batch_size,
"img_h": height_of_image,
"img_w": width_of_image,
"img_chls": num_channels,
"num_cls": num_classes,
"model": model
}
# getting images paths
self.train_paths = glob.glob(os.path.join(train_images_dir,
"**/*.png"),
recursive=True)
self.val_paths = glob.glob(os.path.join(val_images_dir, "**/*.png"),
recursive=True)
self.test_paths = glob.glob(os.path.join(test_images_dir, "**/*.png"),
recursive=True)
# download and load images.
x_train = self.load_images(filename_x_train, train_images_dir)
y_train_cls = self.load_cls(filename_y_train, train_images_dir)
self.x_test = self.load_images(filename_x_test, test_images_dir)
self.y_test_cls = self.load_cls(filename_y_test, test_images_dir)
# split into train/validation sets
self.x_train = x_train[0:self.config["m_train"]]
self.x_val = x_train[self.config["m_train"]:]
self.y_train_cls = y_train_cls[0:self.config["m_train"]]
self.y_val_cls = y_train_cls[self.config["m_train"]:]
# one-hot-encode labels
self.y_train = one_hot_encoded(self.y_train_cls,
self.config["num_cls"])
self.y_val = one_hot_encoded(self.y_val_cls, self.config["num_cls"])
self.y_test = one_hot_encoded(self.y_test_cls, self.config["num_cls"])
# save in class folders
save_in_folders(path=train_images_dir,
images=self.x_train,
labels=self.y_train_cls)
save_in_folders(path=val_images_dir,
images=self.x_val,
labels=self.y_val_cls)
save_in_folders(path=test_images_dir,
images=self.x_test,
labels=self.y_test_cls)
print("----Data Loader init----")
def load_test_data():
"""
Load all the test-data for the CIFAR-10 data-set.
Returns the images, class-numbers and one-hot encoded class-labels.
"""
images, cls = _load_data(filename="test_batch")
return images, cls, one_hot_encoded(class_numbers=cls,
num_classes=num_classes)
########################################################################
def load_training_data():
"""
Load all the training-data for the CIFAR-10 data-set.
The data-set is split into 5 data-files which are merged here.
Returns the images, class-numbers and one-hot encoded class-labels.
"""
# Pre-allocate the arrays for the images and class-numbers for efficiency.
images = np.zeros(
shape=[_num_images_train, img_size, img_size, num_channels],
dtype=float)
cls = np.zeros(shape=[_num_images_train], dtype=int)
# Begin-index for the current batch.
begin = 0
# For each data-file.
for i in range(_num_files_train):
# Load the images and class-numbers from the data-file.
images_batch, cls_batch = _load_data(filename="data_batch_" +
str(i + 1))
# Number of images in this batch.
num_images = len(images_batch)
# End-index for the current batch.
end = begin + num_images
# Store the images into the array.
images[begin:end, :] = images_batch
# Store the class-numbers into the array.
cls[begin:end] = cls_batch
# The begin-index for the next batch is the current end-index.
begin = end
return images, cls, one_hot_encoded(class_numbers=cls,
num_classes=num_classes)
def batch_data_loader(self, batch_size, file_paths, index, perm=None):
x_batch = []
y_cls_batch = []
file_paths = np.array(file_paths)
# shuffle file_paths array
if perm is not None:
file_paths = file_paths[perm]
# get mini-batch of paths
file_paths_batch = file_paths[index:index + batch_size]
# reading images from paths
for path in file_paths_batch:
# getting img array
im = imread(path)
#reshape image
if self.config["model"] == 'DNN':
im = im.reshape(self.config["img_h"] * self.config["img_w"] *
self.config["img_chls"])
elif self.config["model"] == 'RNN':
im = im.reshape(self.config["img_h"], self.config["img_w"])
else:
im = im.reshape(self.config["img_h"], self.config["img_w"],
self.config["img_chls"])
x_batch.append(im)
# getting class by splitting path
cls = int(path.split('/')[-2])
y_cls_batch.append(cls)
# converting to np.array and normalizing
x_batch = np.array(x_batch) / 255.
y_cls_batch = np.array(y_cls_batch)
# getting one-hot-encoded labels
y_batch = one_hot_encoded(y_cls_batch, self.config["num_cls"])
return x_batch, y_batch, y_cls_batch
def load_image(self, path):
image = imread(path)
cls = int(path.split('/')[-2])
label = one_hot_encoded(cls, self.config["num_cls"])
return image, label