def test_add_metrics_enforces_uniqueness_if_already_dereferenced(self): metric1 = mock_core.MockMetric() metric1.save() metric2 = mock_core.MockMetric() metric2.save() obj = SimpleExperiment(name="TestSimpleExperiment", metrics=[metric1]) obj.add_metrics([metric1, metric2]) self.assertEqual(obj.metrics, [metric1, metric2]) # check we can still save obj.save()
def setUp(self): system = mock_types.MockSystem() image_source = mock_types.MockImageSource() self.path_manager = PathManager(['~'], '~/tmp') self.metric = mock_types.MockMetric() self.trial_result = mock_types.MockTrialResult( system=system, image_source=image_source, success=False)
def setUpClass(cls): dbconn.connect_to_test_db() dbconn.setup_image_manager() # Create the basic image sources, systems, and metrics. cls.image_collections = [make_image_collection() for _ in range(2)] cls.systems = [mock_types.MockSystem() for _ in range(2)] cls.metrics = [mock_types.MockMetric() for _ in range(2)] cls.unfinished_metrics = [mock_types.MockMetric() for _ in range(2)] for system in cls.systems: system.save() for metric in cls.metrics: metric.save() for metric in cls.unfinished_metrics: metric.save()
def test_load_referenced_models_autoloads_models_that_are_just_ids( self, mock_autoload): system = mock_core.MockSystem() system.save() image_source = mock_core.MockImageSource() image_source.save() metric = mock_core.MockMetric() metric.save() obj = SimpleExperiment(name="TestSimpleExperiment", systems=[system], image_sources=[image_source], metrics=[metric]) obj.save() del obj # Clear existing references, which should reset the references to ids obj = next(SimpleExperiment.objects.all()) self.assertFalse(mock_autoload.called) obj.load_referenced_models() self.assertTrue(mock_autoload.called) self.assertIn(mock.call(VisionSystem, ids=[system.pk]), mock_autoload.call_args_list) self.assertIn(mock.call(ImageSource, ids=[image_source.pk]), mock_autoload.call_args_list) self.assertIn(mock.call(Metric, ids=[metric.pk]), mock_autoload.call_args_list)
def test_stores_and_loads(self): obj = mock_types.MockMetric() obj.save() # Load all the entities all_entities = list(mtr.Metric.objects.all()) self.assertGreaterEqual(len(all_entities), 1) self.assertEqual(all_entities[0], obj) all_entities[0].delete()
def setUpClass(cls): dbconn.connect_to_test_db() cls.system = mock_types.MockSystem() cls.image_source = mock_types.MockImageSource() cls.metric = mock_types.MockMetric() cls.system.save() cls.image_source.save() cls.metric.save() cls.trial_result = mock_types.MockTrialResult(image_source=cls.image_source, system=cls.system, success=True) cls.trial_result.save()
def test_schedule_tasks_discards_old_metric_results( self, mock_task_manager, mock_load_minimal_trial): systems = [mock_core.MockSystem(_id=ObjectId()) for _ in range(3)] systems[0].is_deterministic = mock.create_autospec( systems[0].is_deterministic, return_value=StochasticBehaviour.NON_DETERMINISTIC) image_sources = [ mock_core.MockImageSource(_id=ObjectId()) for _ in range(3) ] metrics = [mock_core.MockMetric(_id=ObjectId()) for _ in range(3)] repeats = 2 trial_results_map = make_trial_map(systems, image_sources, repeats) make_mock_get_run_system_task(mock_task_manager, trial_results_map) patch_load_minimal_trial(mock_load_minimal_trial) old_metric_results = [ mock_core.MockMetricResult(_id=ObjectId(), success=True) ] metric_result_ids = [] def mock_get_measure_trial_task(*_, **__): result_id = ObjectId() metric_result_ids.append(result_id) mock_task = mock.MagicMock() mock_task.is_finished = True mock_task.result_id = result_id return mock_task mock_task_manager.get_measure_trial_task.side_effect = mock_get_measure_trial_task subject = SimpleExperiment(name="TestSimpleExperiment", systems=systems, image_sources=image_sources, metrics=metrics, repeats=repeats, metric_results=old_metric_results) subject.schedule_tasks() with no_auto_dereference(SimpleExperiment): # Results should be all ids, from the measure_all for metric_result_id in metric_result_ids: self.assertIn(metric_result_id, subject.metric_results) for metric_result in old_metric_results: self.assertNotIn(metric_result.pk, subject.metric_results)
def test_schedule_tasks_runs_all_systems_with_all_image_sources_repeatedly( self, mock_task_manager): systems = [mock_core.MockSystem(_id=ObjectId()) for _ in range(3)] systems[0].is_deterministic = mock.create_autospec( systems[0].is_deterministic, return_value=StochasticBehaviour.NON_DETERMINISTIC) image_sources = [ mock_core.MockImageSource(_id=ObjectId()) for _ in range(3) ] metric = mock_core.MockMetric(_id=ObjectId()) repeats = 2 run_cpus = 3 run_gpus = 2 run_memory = '9GB' run_duration = '2:33:44' subject = SimpleExperiment(name="TestSimpleExperiment", systems=systems, image_sources=image_sources, metrics=[metric], repeats=repeats, run_cpus=run_cpus, run_gpus=run_gpus, run_memory=run_memory, run_duration=run_duration) subject.schedule_tasks() for system in systems: for image_source in image_sources: actual_repeats = repeats if system.is_deterministic( ) is not StochasticBehaviour.DETERMINISTIC else 1 for repeat in range(actual_repeats): self.assertIn( mock.call(system=system, image_source=image_source, repeat=repeat, num_cpus=run_cpus, num_gpus=run_gpus, memory_requirements=run_memory, expected_duration=run_duration), mock_task_manager.get_run_system_task.call_args_list)
def setUpClass(cls): dbconn.connect_to_test_db() dbconn.setup_image_manager() # Create the basic image sources, systems, and metrics. cls.image_collections = [make_image_collection() for _ in range(2)] cls.systems = [mock_types.MockSystem() for _ in range(2)] for system in cls.systems: system.save() cls.metrics = [mock_types.MockMetric() for _ in range(2)] for metric in cls.metrics: metric.save() for image_collection in cls.image_collections: for system in cls.systems: trial_result = mock_types.MockTrialResult( system=system, image_source=image_collection, success=True) trial_result.save() cls.trials.append(trial_result)
def test_load_referenced_models_does_nothing_to_models_that_are_already_objects( self, mock_autoload): system = mock_core.MockSystem() system.save() image_source = mock_core.MockImageSource() image_source.save() metric = mock_core.MockMetric() metric.save() obj = SimpleExperiment(name="TestSimpleExperiment", systems=[system], image_sources=[image_source], metrics=[metric]) obj.save() self.assertFalse(mock_autoload.called) obj.load_referenced_models() self.assertFalse(mock_autoload.called)
def test_stores_and_loads(self): system = mock_core.MockSystem() system.save() image_source = mock_core.MockImageSource() image_source.save() metric = mock_core.MockMetric() metric.save() obj = SimpleExperiment(name="TestSimpleExperiment", enabled=True, systems=[system], image_sources=[image_source], metrics=[metric], plots=[]) obj.save() # Load all the entities all_entities = list(SimpleExperiment.objects.all()) self.assertGreaterEqual(len(all_entities), 1) self.assertEqual(all_entities[0], obj)
def test_schedule_tasks_uses_run_system_tasks_correctly( self, mock_task_manager, mock_load_minimal_trial): systems = [mock_core.MockSystem(_id=ObjectId()) for _ in range(1)] image_sources = [ mock_core.MockImageSource(_id=ObjectId()) for _ in range(1) ] metrics = [mock_core.MockMetric(_id=ObjectId()) for _ in range(1)] repeats = 1 trial_results_map = make_trial_map(systems, image_sources, repeats) # Note: This is a separate test because autospecs are expensive in a loop make_mock_get_run_system_task(mock_task_manager, trial_results_map, autospec=True) patch_load_minimal_trial(mock_load_minimal_trial) subject = SimpleExperiment(name="TestSimpleExperiment", systems=systems, image_sources=image_sources, metrics=metrics, repeats=repeats) subject.schedule_tasks()
def setUp(self): # Remove the collections as the start of the test, so that we're sure it's empty Task._mongometa.collection.drop() MetricResult._mongometa.collection.drop() mock_types.MockMetric._mongometa.collection.drop() self.metrics = [mock_types.MockMetric() for _ in range(2)] for metric in self.metrics: metric.save() self.measure_trial_tasks = {} self.metric_results = {} for metric in self.metrics: self.measure_trial_tasks[metric.identifier] = [] self.metric_results[metric.identifier] = [] for trial_result in self.trials: metric_result = mock_types.MockMetricResult( metric=metric, trial_results=[trial_result], success=True) metric_result.save() self.metric_results[metric.identifier].append( metric_result.identifier) task = MeasureTrialTask(metric=metric, trial_results=[trial_result], state=JobState.DONE, result=metric_result) task.save() self.measure_trial_tasks[metric.identifier].append( task.identifier) for trial_result in self.unfinished_trials: task = MeasureTrialTask(metric=metric, trial_results=[trial_result], state=JobState.UNSTARTED) task.save() self.measure_trial_tasks[metric.identifier].append( task.identifier)
def test_schedule_tasks_uses_measure_trial_tasks_correctly( self, mock_task_manager, mock_load_minimal_trial): systems = [mock_core.MockSystem(_id=ObjectId()) for _ in range(1)] image_sources = [ mock_core.MockImageSource(_id=ObjectId()) for _ in range(1) ] metrics = [mock_core.MockMetric(_id=ObjectId()) for _ in range(1)] repeats = 1 trial_results_map = make_trial_map(systems, image_sources, repeats) make_mock_get_run_system_task(mock_task_manager, trial_results_map) patch_load_minimal_trial(mock_load_minimal_trial) metric_result_ids = [] def mock_get_measure_trial_task(*_, **__): result_id = ObjectId() metric_result_ids.append(result_id) mock_task = mock.create_autospec(MeasureTrialTask) mock_task.is_finished = True mock_task.result_id = result_id return mock_task mock_task_manager.get_measure_trial_task.side_effect = mock_get_measure_trial_task subject = SimpleExperiment( name="TestSimpleExperiment", systems=systems, image_sources=image_sources, metrics=metrics, repeats=repeats, ) subject.schedule_tasks() with no_auto_dereference(SimpleExperiment): for metric_result_id in metric_result_ids: self.assertIn(metric_result_id, subject.metric_results)
def test_schedule_tasks_runs_all_systems_with_all_image_sources_with_all_seeds( self, mock_task_manager): systems = [mock_core.MockSystem(_id=ObjectId()) for _ in range(3)] systems[0].is_deterministic = mock.create_autospec( systems[0].is_deterministic, return_value=StochasticBehaviour.SEEDED) image_sources = [ mock_core.MockImageSource(_id=ObjectId()) for _ in range(3) ] metric = mock_core.MockMetric(_id=ObjectId()) repeats = 2 subject = SimpleExperiment(name="TestSimpleExperiment", systems=systems, image_sources=image_sources, metrics=[metric], repeats=repeats, use_seed=True) subject.schedule_tasks() self.assertEqual(repeats, len( subject.seeds)) # The set of seeds should have auto-filled self.assertEqual(len(subject.seeds), len(set( subject.seeds))) # The set of seeds must be distinct for system in systems: for image_source in image_sources: for seed in subject.seeds: self.assertIn( mock.call(system=system, image_source=image_source, repeat=0, seed=seed, num_cpus=mock.ANY, num_gpus=mock.ANY, memory_requirements=mock.ANY, expected_duration=mock.ANY), mock_task_manager.get_run_system_task.call_args_list)
def test_schedule_tasks_measures_all_trial_results( self, mock_task_manager, mock_load_minimal_trial): systems = [mock_core.MockSystem(_id=ObjectId()) for _ in range(3)] systems[0].is_deterministic = mock.create_autospec( systems[0].is_deterministic, return_value=StochasticBehaviour.NON_DETERMINISTIC) image_sources = [ mock_core.MockImageSource(_id=ObjectId()) for _ in range(3) ] metrics = [mock_core.MockMetric(_id=ObjectId()) for _ in range(3)] repeats = 3 measure_cpus = 16 measure_gpus = 22 measure_memory = '88GB' measure_duration = '0:01:01' trial_results_map = make_trial_map(systems, image_sources, repeats) trial_groups = [ trial_group for trials_by_source in trial_results_map.values() for trial_group in trials_by_source.values() ] trials_by_id = { trial_result.pk: trial_result for trial_group in trial_groups for trial_result in trial_group } make_mock_get_run_system_task(mock_task_manager, trial_results_map, autospec=False) patch_load_minimal_trial(mock_load_minimal_trial) def mock_load_minimal_trial_impl(object_id): if object_id in trials_by_id: return trials_by_id[object_id] raise ValueError(f"Unidentified trial result id {object_id}") mock_load_minimal_trial.side_effect = mock_load_minimal_trial_impl subject = SimpleExperiment(name="TestSimpleExperiment", systems=systems, image_sources=image_sources, metrics=metrics, repeats=repeats, measure_cpus=measure_cpus, measure_gpus=measure_gpus, measure_memory=measure_memory, measure_duration=measure_duration) subject.schedule_tasks() for group_idx, trial_group in enumerate(trial_groups): for metric_idx, metric in enumerate(metrics): self.assertIn( mock.call(trial_results=trial_group, metric=metric, num_cpus=measure_cpus, num_gpus=measure_gpus, memory_requirements=measure_memory, expected_duration=measure_duration), mock_task_manager.get_measure_trial_task.call_args_list, "\nCouldn't find measure_trial_task for metric {0}, trial group {1}" .format(metric_idx, group_idx))