def test_mnist() -> None: fashion_mnist = keras.datasets.fashion_mnist _, (test_images, test_labels) = fashion_mnist.load_data() fashion_mnist2 = keras.datasets.fashion_mnist _, (test_images2, test_labels2) = fashion_mnist2.load_data() fashion_mnist3 = keras.datasets.fashion_mnist _, (test_images3, test_labels3) = fashion_mnist3.load_data() assert (test_images == test_images2).all() assert (test_images3 == test_images2).all() sorted1, sorted2, sorted3 = ( np.sort(test_images, axis=0), np.sort(test_images2, axis=0), np.sort(test_images3, axis=0), ) assert (sorted1 == sorted2).all() assert (sorted3 == sorted2).all() index = 500 assert (sorted1[index] == sorted2[index]).all() assert (sorted3[index] == sorted2[index]).all() assert hash_matrix(sorted1[index]) == hash_matrix(sorted2[index]) assert hash_matrix(sorted3[index]) == hash_matrix(sorted2[index])
def validation_hash_table(self) -> ValHTType: if len(self._validation_hash_table) > 0: return self._validation_hash_table for matrix, label in self.validation_set.items(): self._validation_hash_table[hash_matrix(matrix)] = dict( matrix=matrix.numpy(), predicted=label) return self._validation_hash_table
def test_hash_respects_type() -> None: assert hash_matrix(m1_1) != hash_matrix(m1_float)
def test_hash_respects_order() -> None: assert hash_matrix(m1_1) != hash_matrix(m2) assert hash_matrix(m1_1) != hash_matrix(m3)
def test_hash_is_injective() -> None: assert hash_matrix(m1_0) == hash_matrix(m1_1) assert hash_matrix(m4) == hash_matrix(m5)
def test_grade_calculation() -> None: with mock_redis_connection() as (_, __): # Mock inputs and predictions (3x3) images mock_input_matrices = [ to_matrix([1, 2, 3], [4, 5, 6]), to_matrix([7, 8, 9], [10, 11, 12]), to_matrix([13, 14, 15], [16, 17, 18]), to_matrix([19, 20, 21], [22, 23, 24]), ] hashed_mock_input_matrices = { hash_matrix(i): i for i in mock_input_matrices } mock_inputs = np.array(list(hashed_mock_input_matrices.values())) mock_input_hashes = list(hashed_mock_input_matrices.keys()) mock_predictions_correct = { mock_input_hashes[0]: 0, mock_input_hashes[1]: 0, mock_input_hashes[2]: 0, mock_input_hashes[3]: 0, } mock_predictions_okay = { mock_input_hashes[0]: 1, # wrong mock_input_hashes[1]: 0, mock_input_hashes[2]: 0, mock_input_hashes[3]: 0, } mock_predictions_bad = { mock_input_hashes[0]: 1, # wrong mock_input_hashes[1]: 1, # wrong mock_input_hashes[2]: 0, mock_input_hashes[3]: 0, } mock_predictions_very_bad = { mock_input_hashes[0]: 1, # wrong mock_input_hashes[1]: 1, # wrong mock_input_hashes[2]: 1, # wrong mock_input_hashes[3]: 1, # wrong } validation_data = ValidationData.from_numpy( mock_inputs, np.array(np.array([0] * len(mock_input_matrices)))) assert [hash_matrix(m) for m in mock_inputs] == mock_input_hashes assignment_id = "0" mock_assignment = KerasAssignment( name="Mock Exercise 1", identifier=assignment_id, validation_dataset=KerasAssignmentValidationSet( identifier=assignment_id, validation_data=validation_data, validation_set_size=len(hashed_mock_input_matrices.values()), input_selection_strategy=RandomSelectionStrategy(seed=20), ), submission_deadline=datetime.datetime(year=2019, month=12, day=31, hour=23, minute=59), ) if mock_assignment.validation_dataset: mock_assignment.validation_dataset.ingest(Database.assignments) with unittest.mock.patch("kerasltiprovider.utils.Datetime.now", spec=Datetime) as mocked_time: mocked_time.return_value = datetime.datetime(year=2001, month=1, day=1, hour=0, minute=3) assert mock_assignment.validate(mock_predictions_correct) == (1.0, 1.0) assert mock_assignment.validate(mock_predictions_okay) == (0.75, 0.75) assert mock_assignment.validate(mock_predictions_bad) == (0.5, 0.5) assert mock_assignment.validate(mock_predictions_very_bad) == (0, 0)