def load_dataset_into_batches(file_dir_path: str, subset: Subset, subset_size: int, shuffle: bool = False):
op = ops.chain([
ops.vectorize(),
ops.type_cast(np.float32)
])
dataset = PetsDataset(file_dir_path, subset)
return BatchGenerator(dataset, subset_size, shuffle, op)
def main():
data = PetsDataset("/home/helmuth/dlvc/cifar-10-batches-py", Subset.TRAINING)
# ops chain
op = ops.chain([
ops.vectorize(),
ops.type_cast(np.float32),
ops.add(-127.5),
ops.mul(1/127.5),
])
# batch generator #1
bg1 = BatchGenerator(data, len(data), False)
assert(len(bg1) == 1)
# batch generator #2
bg2 = BatchGenerator(data, 500, False, op)
assert(len(bg2) == 16)
# first batch
cnt = 0
for batch in bg2:
cnt += 1
if cnt < 16:
assert(batch.data.shape == (500, 3072))
assert(batch.labels.shape == (500,))
assert(batch.data.dtype == np.float32)
assert(np.issubdtype(batch.labels.dtype, np.integer))
if cnt == 1:
print("First batch, first sample, not shuffled")
print(batch.data[0])
# batch generator #3
bg3 = BatchGenerator(data, 500, True, op)
# run 5 times through first sample of shuffled batch generator
for i in range(5):
it = iter(bg3)
print("First batch, first sample, shuffled")
print(next(it).data[0])
def test_data_transformation(self):
op = ops.chain([ops.vectorize(), ops.type_cast(np.float32)])
dataset = PetsDataset(os.path.join(os.getcwd(), self._data_dir),
Subset.TRAINING)
batch_gen = BatchGenerator(dataset, 100, False, op)
self.assertEqual(len(batch_gen), 80)
iter_gen = iter(batch_gen)
iter_result = next(iter_gen)
self.assertEqual(iter_result.data[0].shape, (3072, ))
self.assertTrue(np.issubdtype(iter_result.data.dtype, np.float32))
def load_dataset(subset: Subset) -> batches.BatchGenerator:
dataset = PetsDataset('../data/cifar-10-batches-py', subset)
op = ops.chain([
ops.hwc2chw(),
ops.add(-127.5),
ops.mul(1 / 127.5),
ops.type_cast(np.float32)
])
return batches.BatchGenerator(dataset, 128, True, op)
def test_train_with_wrong_type_of_labels(self):
op = ops.chain([
ops.vectorize(),
ops.type_cast(np.float32)
])
dataset = PetsDataset(os.path.join(os.getcwd(), self._data_dir), Subset.TRAINING)
batch_gen = BatchGenerator(dataset, 7959, False, op)
batch_iter = iter(batch_gen)
iter_result = next(batch_iter)
classifier = KnnClassifier(10, 3072, 2)
self.assertRaises(TypeError, classifier.train, iter_result.data, [0, 1, 0])
def test_train_with_proper_data(self):
op = ops.chain([
ops.vectorize(),
ops.type_cast(np.float32)
])
dataset = PetsDataset(os.path.join(os.getcwd(), self._data_dir), Subset.TRAINING)
batch_gen = BatchGenerator(dataset, 7959, False, op)
batch_iter = iter(batch_gen)
iter_result = next(batch_iter)
classifier = KnnClassifier(10, 3072, 2)
classifier.train(iter_result.data, iter_result.label)
def test_train_wrong_vector_size_in_data(self):
op = ops.chain([
ops.vectorize(),
ops.type_cast(np.float32)
])
dataset = PetsDataset(os.path.join(os.getcwd(), self._data_dir), Subset.TRAINING)
batch_gen = BatchGenerator(dataset, 7959, False, op)
batch_iter = iter(batch_gen)
iter_result = next(batch_iter)
classifier = KnnClassifier(10, 3072, 2)
changed_data = np.delete(iter_result.data, 100, 1)
self.assertRaises(RuntimeError, classifier.train, changed_data, iter_result.label)
def test_correctness_of_data_for_train(self):
op = ops.chain([
ops.vectorize(),
ops.type_cast(np.float32)
])
dataset = PetsDataset(os.path.join(os.getcwd(), self._data_dir), Subset.TRAINING)
one_batch_gen = BatchGenerator(dataset, 7959, False, op)
self.assertEqual(len(one_batch_gen), 1)
many_batch_gen = BatchGenerator(dataset, 500, False, op)
self.assertEqual(len(many_batch_gen), 16)
reference = [116., 125., 125., 91., 101.]
batch_iter = iter(many_batch_gen)
batch_iter = next(batch_iter)
[self.assertEqual(item, reference[i]) for i, item in enumerate(batch_iter.data[0][:5])]
def load_dataset(subset: Subset, augment=False) -> batches.BatchGenerator:
dataset = PetsDataset('../data/cifar-10-batches-py', subset)
ops_list = []
if augment:
ops_list += [ops.hflip(), ops.rcrop(32, 12, 'constant')]
ops_list += [
ops.mul(1 / 255),
ops.type_cast(np.float32),
# Imagenet:
# ops.normalize( mean=np.array([0.485, 0.456, 0.406]),
# std=np.array([0.229, 0.224, 0.225])),
# Cifar-10:
ops.normalize(mean=np.array([0.41477802, 0.45935813, 0.49693552]),
std=np.array([0.25241926, 0.24699265, 0.25279155])),
ops.hwc2chw()
]
op = ops.chain(ops_list)
return batches.BatchGenerator(dataset, 128, True, op)
def test_predict_with_proper_data(self):
op = ops.chain([
ops.vectorize(),
ops.type_cast(np.float32)
])
dataset_training = PetsDataset(os.path.join(os.getcwd(), self._data_dir), Subset.TRAINING)
dataset_valid = PetsDataset(os.path.join(os.getcwd(), self._data_dir), Subset.VALIDATION)
batch_gen_t = BatchGenerator(dataset_training, 795, False, op)
batch_gen_v = BatchGenerator(dataset_valid, 204, False, op)
batch_iter_t = iter(batch_gen_t)
iter_result_t = next(batch_iter_t)
batch_iter_v = iter(batch_gen_v)
iter_result_v = next(batch_iter_v)
classifier = KnnClassifier(10, 3072, 2)
classifier.train(iter_result_t.data, iter_result_t.label)
results = classifier.predict(iter_result_v.data)
self.assertEqual(len(results), 204)
for result in results:
self.assertEqual(np.sum(result), 1.0)
from dlvc.dataset import Subset
import dlvc.ops as ops
np.random.seed(0)
torch.manual_seed(0)
TrainedModel = namedtuple('TrainedModel', ['model', 'accuracy'])
DATA_PATH = "../cifar-10-batches-py/"
train_data = PetsDataset(DATA_PATH, Subset.TRAINING)
val_data = PetsDataset(DATA_PATH, Subset.VALIDATION)
test_data = PetsDataset(DATA_PATH, Subset.TEST)
op = ops.chain([
ops.type_cast(np.float32),
ops.add(-127.5),
ops.mul(1 / 127.5),
ops.hwc2chw()
])
train_batches = BatchGenerator(train_data, 128, False, op)
val_batches = BatchGenerator(val_data, 128, False, op)
test_batches = BatchGenerator(test_data, 128, False, op)
class Net(nn.Module):
def __init__(self, img_size, num_classes):
super(Net, self).__init__()
self.img_size = img_size
# Instantiate the ReLU nonlinearity
self.relu = nn.ReLU()
TrainedModel = namedtuple('TrainedModel', ['model', 'accuracy'])
# initialize RNG for reproducability
random.seed(42)
np.random.seed(42)
torch.manual_seed(42)
# Step 1: load the data sets (TRAIN, VALIDATION & TEST)
train_data = PetsDataset("../cifar-10-batches-py", Subset.TRAINING)
val_data = PetsDataset("../cifar-10-batches-py", Subset.VALIDATION)
test_data = PetsDataset("../cifar-10-batches-py", Subset.TEST)
# Operations to standardize
op = ops.chain([
ops.vectorize(),
ops.type_cast(np.float32),
ops.add(-127.5),
ops.mul(1/127.5),
])
# Step 2: Create batch generator for each
BATCH_SIZE = 512
train_batches = BatchGenerator(train_data, BATCH_SIZE, True, op)
val_batches = BatchGenerator(val_data, BATCH_SIZE, True, op)
test_batches = BatchGenerator(test_data, BATCH_SIZE, True, op)
def train_model(lr: float, momentum: float) -> TrainedModel:
'''
Trains a linear classifier with a given learning rate (lr) and momentum.
Computes the accuracy on the validation set.
Returns both the trained classifier and accuracy.
'''
import numpy as np
from dlvc.dataset import Subset
from dlvc.datasets.pets import PetsDataset
from dlvc import ops, batches
dataset = PetsDataset('../data/cifar-10-batches-py', Subset.TRAINING)
op = ops.chain([ops.mul(1 / 255), ops.type_cast(np.float32)])
batch_generator = batches.BatchGenerator(dataset, 7959, True, op)
training_images = []
for batch in batch_generator:
training_images.append(batch.data)
training_images = np.array(training_images, dtype=np.float32)
training_images = training_images.reshape(training_images.shape[1:])
train_mean = np.mean(training_images, axis=(0, 1, 2))
train_std = np.std(training_images, axis=(0, 1, 2))
print(train_mean, train_std)
torch.manual_seed(42)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
# Batch size to be used
BATCH_SIZE = 128
# Step 1: load the data sets (TRAIN, VALIDATION)
train_data = PetsDataset("../cifar-10-batches-py", Subset.TRAINING)
val_data = PetsDataset("../cifar-10-batches-py", Subset.VALIDATION)
# Operations to standardize
# First experiment: scale to [-1,1]
op1 = ops.chain([
ops.type_cast(np.float32),
ops.add(-127.5),
ops.mul(1 / 127.5),
ops.hwc2chw()
])
# Second experiment: scale to sample mean=0, sd=1
# calculate average training sample mean & sd
op_calc = ops.chain([ops.type_cast(np.float32), ops.mean_sd()])
# using batch generator (could do it directly but I'm lazy)
train_full_batch_gen = BatchGenerator(train_data, len(train_data), False,
op_calc)
train_full_batch = next(b for b in train_full_batch_gen)
train_mean_sd = np.mean(train_full_batch.data, axis=0)
# create operation to scale
op2 = ops.chain([
ops.type_cast(np.float32),
ops.scale(train_mean_sd[0], train_mean_sd[1]),
ops.hwc2chw()
def load_dataset(subset: Subset) -> batches.BatchGenerator:
dataset = PetsDataset('../data/cifar-10-batches-py', subset)
op = ops.chain([ops.vectorize(), ops.type_cast(np.float32)])
return batches.BatchGenerator(dataset, len(dataset), True, op)
DATA_PATH = "../cifar-10-batches-py/"
RESULTS_FILE = "results.txt"
NR_EPOCHS = 100
EARLY_STOPPING = 10
CUDA = torch.cuda.is_available()
train_data = PetsDataset(DATA_PATH, Subset.TRAINING)
val_data = PetsDataset(DATA_PATH, Subset.VALIDATION)
op_all_augmentation = ops.chain([
ops.type_cast(np.float32),
ops.add(-127.5),
ops.mul(1 / 127.5),
ops.hflip(),
ops.rcrop(32, 4, 'constant'),
ops.add_noise(),
ops.hwc2chw()
])
op_augmentation_crop_flip = ops.chain([
ops.type_cast(np.float32),
ops.add(-127.5),
ops.mul(1 / 127.5),
ops.hflip(),
ops.rcrop(32, 4, 'constant'),
ops.hwc2chw()
])
op_no_augmentation = ops.chain([
from dlvc.dataset import Subset
import dlvc.ops as ops
np.random.seed(0)
torch.manual_seed(0)
DATA_PATH = "../cifar-10-batches-py/"
MODEL_PATH = "best_model.pt"
train_data = PetsDataset(DATA_PATH, Subset.TRAINING)
val_data = PetsDataset(DATA_PATH, Subset.VALIDATION)
op = ops.chain([
ops.type_cast(np.float32),
ops.add(-127.5),
ops.mul(1 / 127.5),
ops.hflip(),
ops.rcrop(32, 4, 'constant'),
ops.add_noise(),
ops.hwc2chw()
])
train_batches = BatchGenerator(train_data, 128, False, op)
val_batches = BatchGenerator(val_data, 128, False, op)
class Net(nn.Module):
def __init__(self, img_size, num_classes):
super(Net, self).__init__()
self.img_size = img_size
# Instantiate the ReLU nonlinearity
torch.manual_seed(42)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
# Batch size to be used
BATCH_SIZE = 128
# Step 1: load the data sets (TRAIN, VALIDATION)
train_data = PetsDataset("../cifar-10-batches-py", Subset.TRAINING)
val_data = PetsDataset("../cifar-10-batches-py", Subset.VALIDATION)
# Operations to standardize
# scale to sample mean=0, sd=1
# calculate average training sample mean & sd
op_calc = ops.chain([
ops.type_cast(np.float32),
ops.mean_sd()
])
# using batch generator (could do it directly but I'm lazy)
train_full_batch_gen = BatchGenerator(
train_data,
len(train_data),
False,
op_calc)
train_full_batch = next(b for b in train_full_batch_gen)
train_mean_sd = np.mean(train_full_batch.data, axis=0)
# create operation to scale
op2 = ops.chain([
ops.type_cast(np.float32),
ops.scale(train_mean_sd[0], train_mean_sd[1]),
ops.hwc2chw()
])
BATCH_SIZE = 128
NUM_CLASSES = 2
EPOCHS = 500
lr = 0.001
# weight decay 0 in this configuration, in part 3 this is changed
wd = 0.0
pets_training = PetsDataset(dir, Subset.TRAINING)
pets_validation = PetsDataset(dir, Subset.VALIDATION)
pets_test = PetsDataset(dir, Subset.TEST)
batchGenerator_training = BatchGenerator(pets_training, BATCH_SIZE, shuffle=True,
op=chain([type_cast(dtype=np.float32),
add(-127.5),
mul(1 / 127.5),
hwc2chw()]))
batchGenerator_validation = BatchGenerator(pets_validation, BATCH_SIZE, shuffle=False,
op=chain([type_cast(dtype=np.float32),
add(-127.5),
mul(1 / 127.5),
hwc2chw()]))
batchGenerator_test = BatchGenerator(pets_test, BATCH_SIZE, shuffle=False,
op=chain([type_cast(dtype=np.float32),
add(-127.5),
mul(1 / 127.5),
hwc2chw()]))
class CatDogNet(nn.Module):
def __init__(self):
wd = 0.00000001
EARLY_STOPPING = True
EARLY_STOPPING_NUM_OF_EPOCHS = 100
USE_DROPOUT = True
pets_training = PetsDataset(dir, Subset.TRAINING)
pets_validation = PetsDataset(dir, Subset.VALIDATION)
batchGenerator_training = BatchGenerator(pets_training,
BATCH_SIZE,
shuffle=True,
op=chain([
type_cast(dtype=np.float32),
add(-127.5),
mul(1 / 127.5),
rcrop(25, 2, 'median'),
hwc2chw()
]))
batchGenerator_validation = BatchGenerator(pets_validation,
BATCH_SIZE,
shuffle=False,
op=chain([
type_cast(dtype=np.float32),
add(-127.5),
mul(1 / 127.5),
hwc2chw()
]))
class CatDogNet(nn.Module):
# skip last batch
if batch_idx == last_batch_idx:
continue
assert batch.data.shape == (500, 3072), "Batch data shape: " + str(
batch.data.shape) + ", expected: (500, 3072)."
assert batch.labels.shape == (500, ), "Batch labels shape: " + str(
batch.labels.shape) + ", expected: (500,)."
batch_idx += 1
# The data type is always np.float32 and the label type is integral (one of the np.int and np.uint variants)
# Implemented: for label type np.uint8 since there is less than 256 labels
batch_generator = BatchGenerator(
dataset_training,
500,
False,
op=chain([vectorize(), type_cast(dtype=np.float32)]))
for batch in batch_generator:
assert batch.data.dtype == np.float32, "Batch data type: " + str(
batch.data.dtype) + ", expected: np.float32."
assert batch.labels.dtype == np.uint8, "Batch labels type: " + str(
batch.labels.dtype) + ", expected: np.uint8."
# The first sample of the first training batch returned without shuffling
# has label 0 ...
batch_generator = BatchGenerator(
dataset_training,
500,
False,
op=chain([type_cast(dtype=np.float32),
vectorize()]))
first_sample_label_unshuffled = None
dataset = PetsDataset('../data/cifar-10-batches-py',
Subset.TEST)
print('Found dataset directory')
print(f'{len(dataset)} samples')
test_sample = dataset[1]
print(f'Index of test sample: {test_sample.idx}')
print(f'Label of test sample: {test_sample.label}')
#cv2.imshow('Sample Image', test_sample.data)
#cv2.waitKey(0)
#cv2.destroyAllWindows()
op = ops.chain([
ops.vectorize(),
ops.type_cast(np.float32)
])
#The number of training batches is 1 if the batch size is set to the number of samples in the dataset
generator = batches.BatchGenerator(dataset, len(dataset), True, op)
print(len(generator))
#The number of training batches is 16 if the batch size is set to 500
generator = batches.BatchGenerator(dataset, 500, True, op)
print(len(generator))
#The data and label shapes are (500, 3072) and (500,), respectively, unless for the last batch
#The data type is always np.float32 and the label type is integral (one of the np.int and np.uint variants)
#generator = batches.BatchGenerator(dataset, 20, False, op)
for i in iter(generator):
pass
def set_parameter(model, freeze_parameters):
if freeze_parameters:
for param in model.parameters():
param.requires_grad = False
if USE_TRANSFER_LEARNING:
# there are two networks to use in transfer learning "resnet" and "alexnet"
net = initialize_transfer_learning_model("resnet", NUM_CLASSES,
FREEZE_CNN_PARAMETERS)
net, input_size = net
pad_mode_for_resizing = 'constant'
op_chain = chain([
type_cast(dtype=np.float32),
add(-127.5),
mul(1 / 127.5),
rcrop(25, 2, 'median'),
resize(input_size, pad_mode_for_resizing),
hwc2chw()
])
else:
net = CatDogNet()
op_chain = chain([
type_cast(dtype=np.float32),
add(-127.5),
mul(1 / 127.5),
rcrop(25, 2, 'median'),
hwc2chw()
])
batchGenerator_training = BatchGenerator(pets_training,
BATCH_SIZE,
# The data and label shapes are (500, 3072) and (500,), respectively, unless for the last batch
batch_generator = BatchGenerator(dataset_training, 500, shuffle=False, op=vectorize())
last_batch_idx = len(batch_generator) - 1
batch_idx = 0
for batch in batch_generator:
# skip last batch
if batch_idx == last_batch_idx:
continue
assert batch.data.shape == (500, 3072), "Batch data shape: " + str(batch.data.shape) + ", expected: (500, 3072)."
assert batch.label.shape == (500,), "Batch labels shape: " + str(batch.label.shape) + ", expected: (500,)."
batch_idx += 1
# The data type is always np.float32 and the label type is integral (one of the np.int and np.uint variants)
# Implemented: for label type np.uint8 since there is less than 256 labels
batch_generator = BatchGenerator(dataset_training, 500, False, op=chain([vectorize(), type_cast(dtype=np.float32)]))
for batch in batch_generator:
assert batch.data.dtype == np.float32, "Batch data type: " + str(batch.data.dtype) + ", expected: np.float32."
assert batch.label.dtype == np.uint8, "Batch labels type: " + str(batch.label.dtype) + ", expected: np.uint8."
# The first sample of the first training batch returned without shuffling
# has label 0 ...
batch_generator = BatchGenerator(dataset_training, 500, False, op=chain([type_cast(dtype=np.float32), vectorize()]))
first_sample_label_unshuffled = None
first_sample_data_unshuffled = None
expected_label = 0
for batch in batch_generator:
for label in batch.label:
first_sample_label_unshuffled = label
break
for data in batch.data:
