def test_result_id_is_result_primary_key(self): result = mock_types.MockMetricResult() result.pk = bson.ObjectId() obj = MeasureTrialTask(metric=self.metric, trial_results=[self.trial_result], state=JobState.DONE, result=result) self.assertEqual(result.pk, obj.result_id)
def setUp(self): # Remove the collections as the start of the test, so that we're sure it's empty Task._mongometa.collection.drop() TrialResult._mongometa.collection.drop() MetricResult._mongometa.collection.drop() # Create run system tasks and trial results self.run_system_tasks = {} self.trial_result_groups = [] for image_collection in self.image_collections: for system in self.systems: trial_result_group = [] for repeat in range(3): trial_result = mock_types.MockTrialResult( system=system, image_source=image_collection, success=True) trial_result.save() trial_result_group.append(trial_result) task = RunSystemTask(system=system, image_source=image_collection, state=JobState.DONE, result=trial_result) task.save() self.run_system_tasks[ trial_result.identifier] = task.identifier self.trial_result_groups.append(trial_result_group) self.measure_trial_tasks = {} self.metric_results = {} for group_id, trial_result_group in enumerate( self.trial_result_groups): self.measure_trial_tasks[group_id] = [] self.metric_results[group_id] = [] for metric in self.metrics: metric_result = mock_types.MockMetricResult( metric=metric, trial_results=trial_result_group, success=True) metric_result.save() self.metric_results[group_id].append(metric_result.identifier) task = MeasureTrialTask(metric=metric, trial_results=trial_result_group, state=JobState.DONE, result=metric_result) task.save() self.measure_trial_tasks[group_id].append(task.identifier) for metric in self.unfinished_metrics: task = MeasureTrialTask(metric=metric, trial_results=trial_result_group, state=JobState.UNSTARTED) task.save() self.measure_trial_tasks[group_id].append(task.identifier)
def test_stores_and_loads(self): obj = mock_types.MockMetricResult( metric=self.metric, trial_results=[self.trial_result], success=True, message='Completed successfully' ) obj.save() # Load all the entities all_entities = list(mtr.MetricResult.objects.all()) self.assertGreaterEqual(len(all_entities), 1) self.assertEqual(all_entities[0], obj) all_entities[0].delete()
def test_required_fields_are_required(self): # missing metric obj = mock_types.MockMetricResult( trial_results=[self.trial_result], success=True, message='Completed successfully' ) with self.assertRaises(ValidationError): obj.save() # Missing trial results obj = mock_types.MockMetricResult( metric=self.metric, success=True, message='Completed successfully' ) with self.assertRaises(ValidationError): obj.save() # empty trial results obj = mock_types.MockMetricResult( metric=self.metric, trial_results=[], success=True, message='Completed successfully' ) with self.assertRaises(ValidationError): obj.save() # missing success obj = mock_types.MockMetricResult( metric=self.metric, trial_results=[self.trial_result], message='Completed successfully' ) with self.assertRaises(ValidationError): obj.save()
def setUpClass(cls): dbconn.connect_to_test_db() cls.system = mock_types.MockSystem() cls.image_source = mock_types.MockImageSource() cls.metric = mock_types.MockMetric() cls.system.save() cls.image_source.save() cls.metric.save() cls.trial_result = mock_types.MockTrialResult( image_source=cls.image_source, system=cls.system, success=True) cls.trial_result.save() cls.metric_result = mock_types.MockMetricResult( metric=cls.metric, trial_results=[cls.trial_result], success=True) cls.metric_result.save()
def test_schedule_tasks_discards_old_metric_results( self, mock_task_manager, mock_load_minimal_trial): systems = [mock_core.MockSystem(_id=ObjectId()) for _ in range(3)] systems[0].is_deterministic = mock.create_autospec( systems[0].is_deterministic, return_value=StochasticBehaviour.NON_DETERMINISTIC) image_sources = [ mock_core.MockImageSource(_id=ObjectId()) for _ in range(3) ] metrics = [mock_core.MockMetric(_id=ObjectId()) for _ in range(3)] repeats = 2 trial_results_map = make_trial_map(systems, image_sources, repeats) make_mock_get_run_system_task(mock_task_manager, trial_results_map) patch_load_minimal_trial(mock_load_minimal_trial) old_metric_results = [ mock_core.MockMetricResult(_id=ObjectId(), success=True) ] metric_result_ids = [] def mock_get_measure_trial_task(*_, **__): result_id = ObjectId() metric_result_ids.append(result_id) mock_task = mock.MagicMock() mock_task.is_finished = True mock_task.result_id = result_id return mock_task mock_task_manager.get_measure_trial_task.side_effect = mock_get_measure_trial_task subject = SimpleExperiment(name="TestSimpleExperiment", systems=systems, image_sources=image_sources, metrics=metrics, repeats=repeats, metric_results=old_metric_results) subject.schedule_tasks() with no_auto_dereference(SimpleExperiment): # Results should be all ids, from the measure_all for metric_result_id in metric_result_ids: self.assertIn(metric_result_id, subject.metric_results) for metric_result in old_metric_results: self.assertNotIn(metric_result.pk, subject.metric_results)
def setUp(self): # Remove the collections as the start of the test, so that we're sure it's empty Task._mongometa.collection.drop() MetricResult._mongometa.collection.drop() self.metric_results = [] self.measure_trial_tasks = {} for metric in self.metrics: for trial_result in self.trials: metric_result = mock_types.MockMetricResult( metric=metric, trial_results=[trial_result], success=True) metric_result.save() self.metric_results.append(metric_result) task = MeasureTrialTask(metric=metric, trial_results=[trial_result], state=JobState.DONE, result=metric_result) task.save() self.measure_trial_tasks[ metric_result.identifier] = task.identifier
def setUp(self): # Remove the collections as the start of the test, so that we're sure it's empty Task._mongometa.collection.drop() MetricResult._mongometa.collection.drop() mock_types.MockMetric._mongometa.collection.drop() self.metrics = [mock_types.MockMetric() for _ in range(2)] for metric in self.metrics: metric.save() self.measure_trial_tasks = {} self.metric_results = {} for metric in self.metrics: self.measure_trial_tasks[metric.identifier] = [] self.metric_results[metric.identifier] = [] for trial_result in self.trials: metric_result = mock_types.MockMetricResult( metric=metric, trial_results=[trial_result], success=True) metric_result.save() self.metric_results[metric.identifier].append( metric_result.identifier) task = MeasureTrialTask(metric=metric, trial_results=[trial_result], state=JobState.DONE, result=metric_result) task.save() self.measure_trial_tasks[metric.identifier].append( task.identifier) for trial_result in self.unfinished_trials: task = MeasureTrialTask(metric=metric, trial_results=[trial_result], state=JobState.UNSTARTED) task.save() self.measure_trial_tasks[metric.identifier].append( task.identifier)
def test_removes_incomplete_measure_results(self): # Make tasks of each type, some unstarted, some running, some complete unstarted = [] running = [] complete = [] for system in self.systems: for image_collection in self.image_collections: trial_result = mock_types.MockTrialResult( system=system, image_source=image_collection, success=True) trial_result.save() for metric in self.metrics: task = MeasureTrialTask(metric=metric, trial_results=[trial_result], state=JobState.UNSTARTED) task.save() unstarted.append(task) task = MeasureTrialTask(metric=metric, trial_results=[trial_result], state=JobState.RUNNING, job_id=10, node_id='this') task.save() running.append(task) metric_result = mock_types.MockMetricResult( metric=metric, trial_results=[trial_result], success=True) metric_result.save() task = MeasureTrialTask(metric=metric, trial_results=[trial_result], state=JobState.DONE, result=metric_result) task.save() complete.append(task) self.assertEqual( len(complete) + len(running) + len(unstarted), Task.objects.all().count()) invalidate.invalidate_incomplete_tasks() self.assertEqual(len(complete), Task.objects.all().count()) for unstarted_task in unstarted: self.assertEqual( 0, Task.objects.raw({ '_id': unstarted_task.pk }).count()) for running_task in running: self.assertEqual( 0, Task.objects.raw({ '_id': running_task.pk }).count()) for complete_task in complete: self.assertEqual( 1, Task.objects.raw({ '_id': complete_task.pk }).count()) # Clean up after ourselves Task.objects.all().delete() MetricResult.objects.all().delete() TrialResult.objects.all().delete()