def test_mnist() -> None:
fashion_mnist = keras.datasets.fashion_mnist
_, (test_images, test_labels) = fashion_mnist.load_data()
fashion_mnist2 = keras.datasets.fashion_mnist
_, (test_images2, test_labels2) = fashion_mnist2.load_data()
fashion_mnist3 = keras.datasets.fashion_mnist
_, (test_images3, test_labels3) = fashion_mnist3.load_data()
assert (test_images == test_images2).all()
assert (test_images3 == test_images2).all()
sorted1, sorted2, sorted3 = (
np.sort(test_images, axis=0),
np.sort(test_images2, axis=0),
np.sort(test_images3, axis=0),
)
assert (sorted1 == sorted2).all()
assert (sorted3 == sorted2).all()
index = 500
assert (sorted1[index] == sorted2[index]).all()
assert (sorted3[index] == sorted2[index]).all()
assert hash_matrix(sorted1[index]) == hash_matrix(sorted2[index])
assert hash_matrix(sorted3[index]) == hash_matrix(sorted2[index])
def validation_hash_table(self) -> ValHTType:
if len(self._validation_hash_table) > 0:
return self._validation_hash_table
for matrix, label in self.validation_set.items():
self._validation_hash_table[hash_matrix(matrix)] = dict(
matrix=matrix.numpy(), predicted=label)
return self._validation_hash_table
def test_hash_respects_type() -> None:
assert hash_matrix(m1_1) != hash_matrix(m1_float)
def test_hash_respects_order() -> None:
assert hash_matrix(m1_1) != hash_matrix(m2)
assert hash_matrix(m1_1) != hash_matrix(m3)
def test_hash_is_injective() -> None:
assert hash_matrix(m1_0) == hash_matrix(m1_1)
assert hash_matrix(m4) == hash_matrix(m5)
def test_grade_calculation() -> None:
with mock_redis_connection() as (_, __):
# Mock inputs and predictions (3x3) images
mock_input_matrices = [
to_matrix([1, 2, 3], [4, 5, 6]),
to_matrix([7, 8, 9], [10, 11, 12]),
to_matrix([13, 14, 15], [16, 17, 18]),
to_matrix([19, 20, 21], [22, 23, 24]),
]
hashed_mock_input_matrices = {
hash_matrix(i): i
for i in mock_input_matrices
}
mock_inputs = np.array(list(hashed_mock_input_matrices.values()))
mock_input_hashes = list(hashed_mock_input_matrices.keys())
mock_predictions_correct = {
mock_input_hashes[0]: 0,
mock_input_hashes[1]: 0,
mock_input_hashes[2]: 0,
mock_input_hashes[3]: 0,
}
mock_predictions_okay = {
mock_input_hashes[0]: 1, # wrong
mock_input_hashes[1]: 0,
mock_input_hashes[2]: 0,
mock_input_hashes[3]: 0,
}
mock_predictions_bad = {
mock_input_hashes[0]: 1, # wrong
mock_input_hashes[1]: 1, # wrong
mock_input_hashes[2]: 0,
mock_input_hashes[3]: 0,
}
mock_predictions_very_bad = {
mock_input_hashes[0]: 1, # wrong
mock_input_hashes[1]: 1, # wrong
mock_input_hashes[2]: 1, # wrong
mock_input_hashes[3]: 1, # wrong
}
validation_data = ValidationData.from_numpy(
mock_inputs, np.array(np.array([0] * len(mock_input_matrices))))
assert [hash_matrix(m) for m in mock_inputs] == mock_input_hashes
assignment_id = "0"
mock_assignment = KerasAssignment(
name="Mock Exercise 1",
identifier=assignment_id,
validation_dataset=KerasAssignmentValidationSet(
identifier=assignment_id,
validation_data=validation_data,
validation_set_size=len(hashed_mock_input_matrices.values()),
input_selection_strategy=RandomSelectionStrategy(seed=20),
),
submission_deadline=datetime.datetime(year=2019,
month=12,
day=31,
hour=23,
minute=59),
)
if mock_assignment.validation_dataset:
mock_assignment.validation_dataset.ingest(Database.assignments)
with unittest.mock.patch("kerasltiprovider.utils.Datetime.now",
spec=Datetime) as mocked_time:
mocked_time.return_value = datetime.datetime(year=2001,
month=1,
day=1,
hour=0,
minute=3)
assert mock_assignment.validate(mock_predictions_correct) == (1.0,
1.0)
assert mock_assignment.validate(mock_predictions_okay) == (0.75,
0.75)
assert mock_assignment.validate(mock_predictions_bad) == (0.5, 0.5)
assert mock_assignment.validate(mock_predictions_very_bad) == (0,
0)