def test_shuffle(dataset: ILSVRCDataset) -> None:
"""Tests that the shuffling flag works as expected. Also tests that
filenames and labels are still properly mapped.
:param dataset: the dataset.
"""
dataset.trim_dataset(DATASET_FRACTION)
x_train_filenames = dataset.partition[TRAIN_KEY]
y_train = dataset.get_labels(x_train_filenames, True, NUM_CLASSES)
train_sequence = ImageDatasetSequence(x_train_filenames,
y=y_train,
batch_size=BATCH_SIZE,
image_target_size=IMAGE_TARGET_SIZE,
batch_augment_fn=None,
batch_format_fn=None,
overfit_single_batch=False,
shuffle_on_epoch_end=True)
img_to_label_before = {}
for batch in train_sequence:
x_batch, y_batch = batch
for i in range(x_batch.shape[0]):
img_data = tuple(x_batch[i].flatten())
label = tuple(y_batch[i])
img_to_label_before[img_data] = label
# Test shuffle.
first_batch_before = train_sequence.__getitem__(0)
train_sequence.on_epoch_end()
first_batch_after = train_sequence.__getitem__(0)
assert (first_batch_before[0] != first_batch_after[0]).any()
# Test filename/label mappings.
for batch in train_sequence:
x_batch, y_batch = batch
for i in range(x_batch.shape[0]):
img_data = tuple(x_batch[i].flatten())
label = tuple(y_batch[i])
assert img_to_label_before[img_data] == label
def get_model(dataset_args: Dict[str, Any],
network_args: Dict[str, Any]) -> ProjectModel:
"""Returns the model.
:param dataset_args: the dataset arguments; see DEFAULT_DATASET_ARGS for
available arguments.
:param network_args: the network arguments; see DEFAULT_NETWORK_ARGS for
available arguments.
:return: the model.
"""
dataset_args = {**DEFAULT_DATASET_ARGS, **dataset_args}
network_args = {**DEFAULT_NETWORK_ARGS, **network_args}
print('Dataset args: {0}'.format(dataset_args))
print('Network args: {0}'.format(network_args))
print('Loading dataset from {0}'.format(DEFAULT_DATASET_PATH))
dataset = ILSVRCDataset(DEFAULT_DATASET_PATH)
if dataset_args['dataset_fraction'] < 1.0:
dataset.trim_dataset(dataset_args['dataset_fraction'])
print('Num training examples: {0}'.format(
dataset.partition[TRAIN_KEY].shape[0]))
print('Num validation examples: {0}'.format(
dataset.partition[VAL_KEY].shape[0]))
print('Num test examples: {0}'.format(
dataset.partition[TEST_KEY].shape[0]))
if network_args['architecture'] == ARCHITECTURE_MLP:
network = MLP(network_args)
elif network_args['architecture'] == ARCHITECTURE_LENET:
network = LeNet(network_args)
else:
raise ValueError('Unrecognized architecture: {0}'.format(
network_args['architecture']))
return ImageModel(dataset, network)
def test_trim_dataset(dataset: ILSVRCDataset) -> None:
"""Tests that the dataset is being trimmed properly. The trimmed
dataset should be shuffled so that the classes retain the same
approximate distribution.
:param dataset: the dataset.
"""
train_size_before = dataset.partition[TRAIN_KEY].shape[0]
val_size_before = dataset.partition[VAL_KEY].shape[0]
test_size_before = dataset.partition[TEST_KEY].shape[0]
train_subset_before = dataset.partition[TRAIN_KEY][:5]
val_subset_before = dataset.partition[VAL_KEY][:5]
test_subset_before = dataset.partition[TEST_KEY][:5]
dataset.trim_dataset(DATASET_FRACTION, trim_val=True, trim_test=False)
train_size_after = dataset.partition[TRAIN_KEY].shape[0]
val_size_after = dataset.partition[VAL_KEY].shape[0]
test_size_after = dataset.partition[TEST_KEY].shape[0]
train_subset_after = dataset.partition[TRAIN_KEY][:5]
val_subset_after = dataset.partition[VAL_KEY][:5]
test_subset_after = dataset.partition[TEST_KEY][:5]
# Check that trimming occurred (or didn't).
assert (train_size_before * (DATASET_FRACTION - DELTA)) < \
train_size_after < \
(train_size_before * (DATASET_FRACTION + DELTA))
assert (val_size_before * (DATASET_FRACTION - DELTA)) < \
val_size_after < \
(val_size_before * (DATASET_FRACTION + DELTA))
assert test_size_before == test_size_after
# Check that the datasets were shuffled (or weren't).
# We're just going to use the first 5 filenames to check for shuffling;
# it's extremely unlikely that all are the same after shuffling.
assert (train_subset_before != train_subset_after).any()
assert (val_subset_before != val_subset_after).any()
assert (test_subset_before == test_subset_after).all()
def test_training_reproducible() -> None:
"""Tests that training results are reproducible."""
set_random_seed(SEED)
dataset_args = {'dataset_fraction': 0.001}
network_args = {'input_shape': (128, 128, 3), 'num_classes': 1000}
train_args = {'epochs': 10, 'batch_size': 32, 'early_stopping': True}
dataset = ILSVRCDataset(DEFAULT_DATASET_PATH)
dataset.trim_dataset(dataset_args['dataset_fraction'])
network = MLP(network_args)
model = ProjectModel(dataset, network)
history = train_model.train_model(model, train_args)
assert str(history.history) == SEED_HISTORY
def test_images(dataset: ILSVRCDataset) -> None:
"""Tests that the sequence output images meet expected standards.
:param dataset: the dataset.
"""
dataset.trim_dataset(DATASET_FRACTION)
x_train_filenames = dataset.partition[TRAIN_KEY]
y_train = dataset.get_labels(x_train_filenames, True, NUM_CLASSES)
train_sequence = ImageDatasetSequence(x_train_filenames,
y=y_train,
batch_size=BATCH_SIZE,
image_target_size=IMAGE_TARGET_SIZE,
batch_augment_fn=None,
batch_format_fn=None,
overfit_single_batch=False,
shuffle_on_epoch_end=True)
# Test that only the last batch is not of length BATCH_SIZE.
# Also test that there are the correct number of batches.
on_last_batch = False
num_batches_seen = 0
for batch in train_sequence:
assert not on_last_batch
x_batch, y_batch = batch
# Take the first image/label pair and check that it meets standards.
# Check that the image is of the right size.
assert x_batch[0].shape == IMAGE_TARGET_SIZE + (3, )
# Check that the image is of the right datatype.
assert x_batch.dtype == np.float32
# Check that the image is normalized.
assert (0.0 <= x_batch.flatten()).all()
assert (x_batch.flatten() <= 1.0).all()
# Check that the label is categorical and of the right dimension.
assert y_batch.shape[1] == NUM_CLASSES
# Check that the label is of the right datatype.
assert y_batch.dtype == np.float32
# Check that the label is one-hot.
for label in y_batch:
assert sum(label) == 1
on_last_batch = not (x_batch.shape[0] == BATCH_SIZE
and y_batch.shape[0] == BATCH_SIZE)
num_batches_seen += 1
assert num_batches_seen == len(train_sequence)
def test_overfit_single_batch(dataset: ILSVRCDataset) -> None:
"""Tests that the same batch of images is always presented to the
model if overfitting on a single batch.
:param dataset: the dataset.
"""
dataset.trim_dataset(DATASET_FRACTION)
x_train_filenames = dataset.partition[TRAIN_KEY]
y_train = dataset.get_labels(x_train_filenames, True, NUM_CLASSES)
# Test that you can't set overfit and shuffle flags together.
train_sequence = ImageDatasetSequence(x_train_filenames,
y=y_train,
batch_size=BATCH_SIZE,
image_target_size=IMAGE_TARGET_SIZE,
batch_augment_fn=None,
batch_format_fn=None,
overfit_single_batch=True,
shuffle_on_epoch_end=True)
with pytest.raises(ValueError):
for _ in train_sequence:
pass
# Test that you always get the same batch, even after multiple epochs.
train_sequence = ImageDatasetSequence(x_train_filenames,
y=y_train,
batch_size=BATCH_SIZE,
image_target_size=IMAGE_TARGET_SIZE,
batch_augment_fn=None,
batch_format_fn=None,
overfit_single_batch=True,
shuffle_on_epoch_end=False)
num_batches_epoch_1 = 0
for batch in train_sequence:
assert (batch[0] == train_sequence.__getitem__(0)[0]).all()
num_batches_epoch_1 += 1
train_sequence.on_epoch_end()
num_batches_epoch_2 = 0
for batch in train_sequence:
assert (batch[0] == train_sequence.__getitem__(0)[0]).all()
num_batches_epoch_2 += 1
assert num_batches_epoch_1 == num_batches_epoch_2
def dataset() -> ILSVRCDataset:
"""Returns an ILSVRCDataset.
:return: the dataset.
"""
dataset = ILSVRCDataset(DEFAULT_DATASET_PATH)
return dataset