def test_returns_different_error_messages_if_systems_or_image_sources_are_different(self): system1 = mock_types.MockSystem() system2 = mock_types.MockSystem() image_source1 = mock_types.MockImageSource() image_source2 = mock_types.MockImageSource() group1 = [ mock_types.MockTrialResult(image_source=image_source1, system=system1, success=True) for _ in range(10) ] group2 = [ mock_types.MockTrialResult(image_source=image_source2, system=system1, success=True) for _ in range(10) ] group3 = [ mock_types.MockTrialResult(image_source=image_source1, system=system2, success=True) for _ in range(10) ] different_image_source_msg = mtr.check_trial_collection(group1 + group2) self.assertIn('image source', different_image_source_msg) different_system_msg = mtr.check_trial_collection(group1 + group3) self.assertNotEqual(different_system_msg, different_image_source_msg) self.assertIn('system', different_system_msg)
def test_add_vision_systems_enforces_uniqueness_if_already_dereferenced( self): system1 = mock_core.MockSystem() system1.save() system2 = mock_core.MockSystem() system2.save() obj = SimpleExperiment(name="TestSimpleExperiment", systems=[system1]) obj.add_vision_systems([system1, system2]) self.assertEqual(obj.systems, [system1, system2]) # check we can still save obj.save()
def test_returns_error_message_if_trials_have_different_systems(self): system1 = mock_types.MockSystem() system2 = mock_types.MockSystem() image_source = mock_types.MockImageSource() group1 = [ mock_types.MockTrialResult(image_source=image_source, system=system1, success=True) for _ in range(10) ] group2 = [ mock_types.MockTrialResult(image_source=image_source, system=system2, success=True) for _ in range(10) ] message = mtr.check_trial_collection(group1 + group2) self.assertIsNotNone(message) self.assertIn('system', message)
def test_load_referenced_models_autoloads_models_that_are_just_ids( self, mock_autoload): system = mock_core.MockSystem() system.save() image_source = mock_core.MockImageSource() image_source.save() metric = mock_core.MockMetric() metric.save() obj = SimpleExperiment(name="TestSimpleExperiment", systems=[system], image_sources=[image_source], metrics=[metric]) obj.save() del obj # Clear existing references, which should reset the references to ids obj = next(SimpleExperiment.objects.all()) self.assertFalse(mock_autoload.called) obj.load_referenced_models() self.assertTrue(mock_autoload.called) self.assertIn(mock.call(VisionSystem, ids=[system.pk]), mock_autoload.call_args_list) self.assertIn(mock.call(ImageSource, ids=[image_source.pk]), mock_autoload.call_args_list) self.assertIn(mock.call(Metric, ids=[metric.pk]), mock_autoload.call_args_list)
def setUp(self): system = mock_types.MockSystem() image_source = mock_types.MockImageSource() self.path_manager = PathManager(['~'], '~/tmp') self.metric = mock_types.MockTrialComparisonMetric() self.trial_result_1 = mock_types.MockTrialResult(system=system, image_source=image_source, success=False) self.trial_result_2 = mock_types.MockTrialResult(system=system, image_source=image_source, success=False)
def setUpClass(cls): cls.system = mock_types.MockSystem() cls.image_source = mock_types.MockImageSource() image_data = [ np.random.normal(128, 20, size=(10, 10)).astype(np.uint8) for _ in range(cls.num_images) ] cls.images = [ Image( pixels=pixels, metadata=make_metadata( pixels=pixels, source_type=ImageSourceType.SYNTHETIC, camera_pose=Transform( (idx * 15, idx, 0), tf3d.quaternions.axangle2quat((1, 2, 3), 5 * idx * np.pi / (2 * cls.num_images)), w_first=True ), intrinsics=CameraIntrinsics( width=10, height=10, fx=5, fy=5, cx=5, cy=5 ) ) ) for idx, pixels in enumerate(image_data) ]
def test_returns_none_if_all_trials_succeed_and_have_same_image_source_and_system(self): system = mock_types.MockSystem() image_source = mock_types.MockImageSource() group1 = [ mock_types.MockTrialResult(image_source=image_source, system=system, success=True) for _ in range(10) ] self.assertIsNone(mtr.check_trial_collection(group1))
def setUpClass(cls): dbconn.connect_to_test_db() cls.system = mock_types.MockSystem() cls.image_source = mock_types.MockImageSource() cls.system.save() cls.image_source.save() cls.trial_result = mock_types.MockTrialResult(image_source=cls.image_source, system=cls.system, success=True) cls.trial_result.save()
def setUpClass(cls): dbconn.connect_to_test_db() dbconn.setup_image_manager() cls.path_manager = PathManager(['~'], '~/tmp') cls.system = mock_types.MockSystem() cls.image_source = mock_types.MockImageSource() cls.image_source.build_images() cls.system.save() cls.image_source.save()
def setUpClass(cls): dbconn.connect_to_test_db() dbconn.setup_image_manager() # Create the basic image sources, systems, and metrics. cls.image_collections = [make_image_collection() for _ in range(2)] cls.systems = [mock_types.MockSystem() for _ in range(2)] for system in cls.systems: system.save() cls.metrics = [mock_types.MockMetric() for _ in range(2)] for metric in cls.metrics: metric.save()
def setUpClass(cls): dbconn.connect_to_test_db() cls.system = mock_types.MockSystem() cls.image_source = mock_types.MockImageSource() cls.metric = mock_types.MockTrialComparisonMetric() cls.system.save() cls.image_source.save() cls.metric.save() cls.trial_result_1 = mock_types.MockTrialResult( image_source=cls.image_source, system=cls.system, success=True) cls.trial_result_1.save() cls.trial_result_2 = mock_types.MockTrialResult( image_source=cls.image_source, system=cls.system, success=True) cls.trial_result_2.save()
def setUp(self): # Remove the collections as the start of the test, so that we're sure it's empty Task._mongometa.collection.drop() Image._mongometa.collection.drop() TrialResult._mongometa.collection.drop() mock_types.MockSystem._mongometa.collection.drop() # Create the basic image sources in two groups self.image_collections = [make_image_collection() for _ in range(3)] self.unfinisned_image_collections = [ make_image_collection() for _ in range(3) ] # Make systesm self.systems = [mock_types.MockSystem() for _ in range(2)] # Create run system tasks and trial results self.run_system_tasks = {} self.trial_results = {} for system in self.systems: system.save() self.run_system_tasks[system.identifier] = [] self.trial_results[system.identifier] = [] for image_collection in self.image_collections: trial_result = mock_types.MockTrialResult( system=system, image_source=image_collection, success=True) trial_result.save() self.trial_results[system.identifier].append( trial_result.identifier) task = RunSystemTask(system=system, image_source=image_collection, state=JobState.DONE, result=trial_result) task.save() self.run_system_tasks[system.identifier].append( task.identifier) for image_collection in self.unfinisned_image_collections: task = RunSystemTask( system=system, image_source=image_collection, state=JobState.UNSTARTED, ) task.save() self.run_system_tasks[system.identifier].append( task.identifier)
def test_schedule_tasks_discards_old_metric_results( self, mock_task_manager, mock_load_minimal_trial): systems = [mock_core.MockSystem(_id=ObjectId()) for _ in range(3)] systems[0].is_deterministic = mock.create_autospec( systems[0].is_deterministic, return_value=StochasticBehaviour.NON_DETERMINISTIC) image_sources = [ mock_core.MockImageSource(_id=ObjectId()) for _ in range(3) ] metrics = [mock_core.MockMetric(_id=ObjectId()) for _ in range(3)] repeats = 2 trial_results_map = make_trial_map(systems, image_sources, repeats) make_mock_get_run_system_task(mock_task_manager, trial_results_map) patch_load_minimal_trial(mock_load_minimal_trial) old_metric_results = [ mock_core.MockMetricResult(_id=ObjectId(), success=True) ] metric_result_ids = [] def mock_get_measure_trial_task(*_, **__): result_id = ObjectId() metric_result_ids.append(result_id) mock_task = mock.MagicMock() mock_task.is_finished = True mock_task.result_id = result_id return mock_task mock_task_manager.get_measure_trial_task.side_effect = mock_get_measure_trial_task subject = SimpleExperiment(name="TestSimpleExperiment", systems=systems, image_sources=image_sources, metrics=metrics, repeats=repeats, metric_results=old_metric_results) subject.schedule_tasks() with no_auto_dereference(SimpleExperiment): # Results should be all ids, from the measure_all for metric_result_id in metric_result_ids: self.assertIn(metric_result_id, subject.metric_results) for metric_result in old_metric_results: self.assertNotIn(metric_result.pk, subject.metric_results)
def test_schedule_tasks_runs_all_systems_with_all_image_sources_repeatedly( self, mock_task_manager): systems = [mock_core.MockSystem(_id=ObjectId()) for _ in range(3)] systems[0].is_deterministic = mock.create_autospec( systems[0].is_deterministic, return_value=StochasticBehaviour.NON_DETERMINISTIC) image_sources = [ mock_core.MockImageSource(_id=ObjectId()) for _ in range(3) ] metric = mock_core.MockMetric(_id=ObjectId()) repeats = 2 run_cpus = 3 run_gpus = 2 run_memory = '9GB' run_duration = '2:33:44' subject = SimpleExperiment(name="TestSimpleExperiment", systems=systems, image_sources=image_sources, metrics=[metric], repeats=repeats, run_cpus=run_cpus, run_gpus=run_gpus, run_memory=run_memory, run_duration=run_duration) subject.schedule_tasks() for system in systems: for image_source in image_sources: actual_repeats = repeats if system.is_deterministic( ) is not StochasticBehaviour.DETERMINISTIC else 1 for repeat in range(actual_repeats): self.assertIn( mock.call(system=system, image_source=image_source, repeat=repeat, num_cpus=run_cpus, num_gpus=run_gpus, memory_requirements=run_memory, expected_duration=run_duration), mock_task_manager.get_run_system_task.call_args_list)
def setUpClass(cls): dbconn.connect_to_test_db() dbconn.setup_image_manager() # Create the basic image sources, systems, and metrics. cls.image_collections = [make_image_collection() for _ in range(2)] cls.systems = [mock_types.MockSystem() for _ in range(2)] for system in cls.systems: system.save() cls.metrics = [mock_types.MockMetric() for _ in range(2)] for metric in cls.metrics: metric.save() for image_collection in cls.image_collections: for system in cls.systems: trial_result = mock_types.MockTrialResult( system=system, image_source=image_collection, success=True) trial_result.save() cls.trials.append(trial_result)
def test_load_referenced_models_does_nothing_to_models_that_are_already_objects( self, mock_autoload): system = mock_core.MockSystem() system.save() image_source = mock_core.MockImageSource() image_source.save() metric = mock_core.MockMetric() metric.save() obj = SimpleExperiment(name="TestSimpleExperiment", systems=[system], image_sources=[image_source], metrics=[metric]) obj.save() self.assertFalse(mock_autoload.called) obj.load_referenced_models() self.assertFalse(mock_autoload.called)
def test_schedule_tasks_uses_run_system_tasks_correctly( self, mock_task_manager, mock_load_minimal_trial): systems = [mock_core.MockSystem(_id=ObjectId()) for _ in range(1)] image_sources = [ mock_core.MockImageSource(_id=ObjectId()) for _ in range(1) ] metrics = [mock_core.MockMetric(_id=ObjectId()) for _ in range(1)] repeats = 1 trial_results_map = make_trial_map(systems, image_sources, repeats) # Note: This is a separate test because autospecs are expensive in a loop make_mock_get_run_system_task(mock_task_manager, trial_results_map, autospec=True) patch_load_minimal_trial(mock_load_minimal_trial) subject = SimpleExperiment(name="TestSimpleExperiment", systems=systems, image_sources=image_sources, metrics=metrics, repeats=repeats) subject.schedule_tasks()
def test_stores_and_loads(self): system = mock_core.MockSystem() system.save() image_source = mock_core.MockImageSource() image_source.save() metric = mock_core.MockMetric() metric.save() obj = SimpleExperiment(name="TestSimpleExperiment", enabled=True, systems=[system], image_sources=[image_source], metrics=[metric], plots=[]) obj.save() # Load all the entities all_entities = list(SimpleExperiment.objects.all()) self.assertGreaterEqual(len(all_entities), 1) self.assertEqual(all_entities[0], obj)
def test_schedule_tasks_runs_all_systems_with_all_image_sources_with_all_seeds( self, mock_task_manager): systems = [mock_core.MockSystem(_id=ObjectId()) for _ in range(3)] systems[0].is_deterministic = mock.create_autospec( systems[0].is_deterministic, return_value=StochasticBehaviour.SEEDED) image_sources = [ mock_core.MockImageSource(_id=ObjectId()) for _ in range(3) ] metric = mock_core.MockMetric(_id=ObjectId()) repeats = 2 subject = SimpleExperiment(name="TestSimpleExperiment", systems=systems, image_sources=image_sources, metrics=[metric], repeats=repeats, use_seed=True) subject.schedule_tasks() self.assertEqual(repeats, len( subject.seeds)) # The set of seeds should have auto-filled self.assertEqual(len(subject.seeds), len(set( subject.seeds))) # The set of seeds must be distinct for system in systems: for image_source in image_sources: for seed in subject.seeds: self.assertIn( mock.call(system=system, image_source=image_source, repeat=0, seed=seed, num_cpus=mock.ANY, num_gpus=mock.ANY, memory_requirements=mock.ANY, expected_duration=mock.ANY), mock_task_manager.get_run_system_task.call_args_list)
def test_schedule_tasks_uses_measure_trial_tasks_correctly( self, mock_task_manager, mock_load_minimal_trial): systems = [mock_core.MockSystem(_id=ObjectId()) for _ in range(1)] image_sources = [ mock_core.MockImageSource(_id=ObjectId()) for _ in range(1) ] metrics = [mock_core.MockMetric(_id=ObjectId()) for _ in range(1)] repeats = 1 trial_results_map = make_trial_map(systems, image_sources, repeats) make_mock_get_run_system_task(mock_task_manager, trial_results_map) patch_load_minimal_trial(mock_load_minimal_trial) metric_result_ids = [] def mock_get_measure_trial_task(*_, **__): result_id = ObjectId() metric_result_ids.append(result_id) mock_task = mock.create_autospec(MeasureTrialTask) mock_task.is_finished = True mock_task.result_id = result_id return mock_task mock_task_manager.get_measure_trial_task.side_effect = mock_get_measure_trial_task subject = SimpleExperiment( name="TestSimpleExperiment", systems=systems, image_sources=image_sources, metrics=metrics, repeats=repeats, ) subject.schedule_tasks() with no_auto_dereference(SimpleExperiment): for metric_result_id in metric_result_ids: self.assertIn(metric_result_id, subject.metric_results)
def setUpClass(cls): dbconn.connect_to_test_db() cls.system = mock_types.MockSystem() cls.image_source = mock_types.MockImageSource() cls.system.save() cls.image_source.save()
def test_schedule_tasks_measures_all_trial_results( self, mock_task_manager, mock_load_minimal_trial): systems = [mock_core.MockSystem(_id=ObjectId()) for _ in range(3)] systems[0].is_deterministic = mock.create_autospec( systems[0].is_deterministic, return_value=StochasticBehaviour.NON_DETERMINISTIC) image_sources = [ mock_core.MockImageSource(_id=ObjectId()) for _ in range(3) ] metrics = [mock_core.MockMetric(_id=ObjectId()) for _ in range(3)] repeats = 3 measure_cpus = 16 measure_gpus = 22 measure_memory = '88GB' measure_duration = '0:01:01' trial_results_map = make_trial_map(systems, image_sources, repeats) trial_groups = [ trial_group for trials_by_source in trial_results_map.values() for trial_group in trials_by_source.values() ] trials_by_id = { trial_result.pk: trial_result for trial_group in trial_groups for trial_result in trial_group } make_mock_get_run_system_task(mock_task_manager, trial_results_map, autospec=False) patch_load_minimal_trial(mock_load_minimal_trial) def mock_load_minimal_trial_impl(object_id): if object_id in trials_by_id: return trials_by_id[object_id] raise ValueError(f"Unidentified trial result id {object_id}") mock_load_minimal_trial.side_effect = mock_load_minimal_trial_impl subject = SimpleExperiment(name="TestSimpleExperiment", systems=systems, image_sources=image_sources, metrics=metrics, repeats=repeats, measure_cpus=measure_cpus, measure_gpus=measure_gpus, measure_memory=measure_memory, measure_duration=measure_duration) subject.schedule_tasks() for group_idx, trial_group in enumerate(trial_groups): for metric_idx, metric in enumerate(metrics): self.assertIn( mock.call(trial_results=trial_group, metric=metric, num_cpus=measure_cpus, num_gpus=measure_gpus, memory_requirements=measure_memory, expected_duration=measure_duration), mock_task_manager.get_measure_trial_task.call_args_list, "\nCouldn't find measure_trial_task for metric {0}, trial group {1}" .format(metric_idx, group_idx))
def setUp(self): self.system = mock_types.MockSystem() self.image_source = mock_types.MockImageSource() self.image_source.build_images(10) self.path_manager = PathManager(['~'], '~/tmp')
def setUp(self): # Remove the collections as the start of the test, so that we're sure it's empty Task._mongometa.collection.drop() Image._mongometa.collection.drop() ImageCollection._mongometa.collection.drop() TrialResult._mongometa.collection.drop() # Create systems in two groups self.systems = [mock_types.MockSystem() for _ in range(2)] for system in self.systems: system.save() self.unfinished_systems = [mock_types.MockSystem() for _ in range(2)] for system in self.unfinished_systems: system.save() # Make image collections with import tasks self.image_collections = [] self.import_dataset_tasks = {} for _ in range(2): image_collection = make_image_collection() image_collection.save() self.image_collections.append(image_collection) task = ImportDatasetTask(module_name='myloader', path='test/filename', result=image_collection, state=JobState.DONE) task.save() self.import_dataset_tasks[ image_collection.identifier] = task.identifier # Create run system tasks and trial results on each image collection self.run_system_tasks = {} self.trial_results = {} for image_collection in self.image_collections: self.run_system_tasks[image_collection.identifier] = [] self.trial_results[image_collection.identifier] = [] for system in self.systems: trial_result = mock_types.MockTrialResult( system=system, image_source=image_collection, success=True) trial_result.save() self.trial_results[image_collection.identifier].append( trial_result.identifier) task = RunSystemTask(system=system, image_source=image_collection, state=JobState.DONE, result=trial_result) task.save() self.run_system_tasks[image_collection.identifier].append( task.identifier) for system in self.unfinished_systems: task = RunSystemTask( system=system, image_source=image_collection, state=JobState.UNSTARTED, ) task.save() self.run_system_tasks[image_collection.identifier].append( task.identifier)