def test_fit_predict_dump_load(self): """Fits a model, tests that it predicts correctly, dumps and loads it, then tests that it still predicts""" model = Model(self.pipeline) # Test attempting to predict before fitting with self.assertRaises(RuntimeError): model.predict('Lorem ipsum dolor sit amet.') model.fit(self.dataset, groundtruth_directory=self.groundtruth_2_directory) # Test X and y data are set self.assertTrue(model.X_data) self.assertTrue(model.y_data) # Test that there is at least one prediction resulting_ann = model.predict( 'To exclude the possibility that alterations in PSSD might be a consequence of changes in the volume of reference, we used a subset of the vibratome sections' ) self.assertIsInstance(resulting_ann, Annotations) self.assertTrue(resulting_ann) # Test prediction over directory resulting_dataset = model.predict( self.dataset.data_directory, prediction_directory=self.prediction_directory) self.assertIsInstance(resulting_dataset, Dataset) self.assertEqual(len(self.dataset), len(resulting_dataset)) # Test that groundtruth is written groundtruth_dataset = Dataset(self.groundtruth_2_directory) expected = [d.file_name for d in self.dataset] actual = [d.file_name for d in groundtruth_dataset] self.assertListEqual(expected, actual) # Test that the groundtruth ann files have content for ann in groundtruth_dataset.generate_annotations(): self.assertTrue(ann) # Test pickling a model pickle_path = os.path.join(self.prediction_directory, 'test.pkl') model.dump(pickle_path) new_model = Model(self.pipeline) new_model.load(pickle_path) # Test that there is at least one prediction resulting_ann = new_model.predict( 'To exclude the possibility that alterations in PSSD might be a consequence of changes in the volume of reference, we used a subset of the vibratome sections' ) self.assertIsInstance(resulting_ann, Annotations) self.assertTrue(resulting_ann)
def test_cross_validate_create_groundtruth_predictions(self): """ Tests that during cross validation, the medaCy groundtruth (that is, the version of the training dataset used by medaCy) is written as well as the predictions that are created for each fold """ model = Model(self.pipeline) model.cross_validate(self.dataset, num_folds=2, prediction_directory=self.prediction_directory_3, groundtruth_directory=self.groundtruth_directory) prediction_dataset = Dataset(self.prediction_directory_3) groundtruth_dataset = Dataset(self.groundtruth_directory) for d in [prediction_dataset, groundtruth_dataset]: self.assertIsInstance(d, Dataset) original_file_names = {d.file_name for d in self.dataset} prediction_file_names = {d.file_name for d in prediction_dataset} groundtruth_file_names = {d.file_name for d in groundtruth_dataset} for n in [prediction_file_names, groundtruth_file_names]: self.assertSetEqual(n, original_file_names) # Container for all Annotations in all files in all folds all_anns_all_folds_actual = Annotations([]) # Test that fold groundtruth is written to file for fold_name in ["fold_1", "fold_2"]: fold_dataset = Dataset(groundtruth_dataset.data_directory / fold_name) for d in fold_dataset: fold_ann = Annotations(d.ann_path) groundtruth_ann = groundtruth_dataset[d.file_name] # Test that the entities in the fold groundtruth are a subset of the whole for that file self.assertTrue(set(fold_ann) <= set(groundtruth_ann)) all_anns_all_folds_actual |= fold_ann # Container for all annotations pulled directly from the groundtruth dataset all_groundtruth_tuples = Annotations([]) for ann in groundtruth_dataset.generate_annotations(): all_groundtruth_tuples |= ann expected = set(all_groundtruth_tuples) actual = set(all_anns_all_folds_actual) self.assertSetEqual(expected, actual)