def test_result_id_is_result_primary_key(self):
result = mock_types.MockMetricResult()
result.pk = bson.ObjectId()
obj = MeasureTrialTask(metric=self.metric,
trial_results=[self.trial_result],
state=JobState.DONE,
result=result)
self.assertEqual(result.pk, obj.result_id)
def setUp(self):
# Remove the collections as the start of the test, so that we're sure it's empty
Task._mongometa.collection.drop()
TrialResult._mongometa.collection.drop()
MetricResult._mongometa.collection.drop()
# Create run system tasks and trial results
self.run_system_tasks = {}
self.trial_result_groups = []
for image_collection in self.image_collections:
for system in self.systems:
trial_result_group = []
for repeat in range(3):
trial_result = mock_types.MockTrialResult(
system=system,
image_source=image_collection,
success=True)
trial_result.save()
trial_result_group.append(trial_result)
task = RunSystemTask(system=system,
image_source=image_collection,
state=JobState.DONE,
result=trial_result)
task.save()
self.run_system_tasks[
trial_result.identifier] = task.identifier
self.trial_result_groups.append(trial_result_group)
self.measure_trial_tasks = {}
self.metric_results = {}
for group_id, trial_result_group in enumerate(
self.trial_result_groups):
self.measure_trial_tasks[group_id] = []
self.metric_results[group_id] = []
for metric in self.metrics:
metric_result = mock_types.MockMetricResult(
metric=metric,
trial_results=trial_result_group,
success=True)
metric_result.save()
self.metric_results[group_id].append(metric_result.identifier)
task = MeasureTrialTask(metric=metric,
trial_results=trial_result_group,
state=JobState.DONE,
result=metric_result)
task.save()
self.measure_trial_tasks[group_id].append(task.identifier)
for metric in self.unfinished_metrics:
task = MeasureTrialTask(metric=metric,
trial_results=trial_result_group,
state=JobState.UNSTARTED)
task.save()
self.measure_trial_tasks[group_id].append(task.identifier)
def test_stores_and_loads(self):
obj = mock_types.MockMetricResult(
metric=self.metric,
trial_results=[self.trial_result],
success=True,
message='Completed successfully'
)
obj.save()
# Load all the entities
all_entities = list(mtr.MetricResult.objects.all())
self.assertGreaterEqual(len(all_entities), 1)
self.assertEqual(all_entities[0], obj)
all_entities[0].delete()
def test_required_fields_are_required(self):
# missing metric
obj = mock_types.MockMetricResult(
trial_results=[self.trial_result],
success=True,
message='Completed successfully'
)
with self.assertRaises(ValidationError):
obj.save()
# Missing trial results
obj = mock_types.MockMetricResult(
metric=self.metric,
success=True,
message='Completed successfully'
)
with self.assertRaises(ValidationError):
obj.save()
# empty trial results
obj = mock_types.MockMetricResult(
metric=self.metric,
trial_results=[],
success=True,
message='Completed successfully'
)
with self.assertRaises(ValidationError):
obj.save()
# missing success
obj = mock_types.MockMetricResult(
metric=self.metric,
trial_results=[self.trial_result],
message='Completed successfully'
)
with self.assertRaises(ValidationError):
obj.save()
def setUpClass(cls):
dbconn.connect_to_test_db()
cls.system = mock_types.MockSystem()
cls.image_source = mock_types.MockImageSource()
cls.metric = mock_types.MockMetric()
cls.system.save()
cls.image_source.save()
cls.metric.save()
cls.trial_result = mock_types.MockTrialResult(
image_source=cls.image_source, system=cls.system, success=True)
cls.trial_result.save()
cls.metric_result = mock_types.MockMetricResult(
metric=cls.metric, trial_results=[cls.trial_result], success=True)
cls.metric_result.save()
def test_schedule_tasks_discards_old_metric_results(
self, mock_task_manager, mock_load_minimal_trial):
systems = [mock_core.MockSystem(_id=ObjectId()) for _ in range(3)]
systems[0].is_deterministic = mock.create_autospec(
systems[0].is_deterministic,
return_value=StochasticBehaviour.NON_DETERMINISTIC)
image_sources = [
mock_core.MockImageSource(_id=ObjectId()) for _ in range(3)
]
metrics = [mock_core.MockMetric(_id=ObjectId()) for _ in range(3)]
repeats = 2
trial_results_map = make_trial_map(systems, image_sources, repeats)
make_mock_get_run_system_task(mock_task_manager, trial_results_map)
patch_load_minimal_trial(mock_load_minimal_trial)
old_metric_results = [
mock_core.MockMetricResult(_id=ObjectId(), success=True)
]
metric_result_ids = []
def mock_get_measure_trial_task(*_, **__):
result_id = ObjectId()
metric_result_ids.append(result_id)
mock_task = mock.MagicMock()
mock_task.is_finished = True
mock_task.result_id = result_id
return mock_task
mock_task_manager.get_measure_trial_task.side_effect = mock_get_measure_trial_task
subject = SimpleExperiment(name="TestSimpleExperiment",
systems=systems,
image_sources=image_sources,
metrics=metrics,
repeats=repeats,
metric_results=old_metric_results)
subject.schedule_tasks()
with no_auto_dereference(SimpleExperiment):
# Results should be all ids, from the measure_all
for metric_result_id in metric_result_ids:
self.assertIn(metric_result_id, subject.metric_results)
for metric_result in old_metric_results:
self.assertNotIn(metric_result.pk, subject.metric_results)
def setUp(self):
# Remove the collections as the start of the test, so that we're sure it's empty
Task._mongometa.collection.drop()
MetricResult._mongometa.collection.drop()
self.metric_results = []
self.measure_trial_tasks = {}
for metric in self.metrics:
for trial_result in self.trials:
metric_result = mock_types.MockMetricResult(
metric=metric, trial_results=[trial_result], success=True)
metric_result.save()
self.metric_results.append(metric_result)
task = MeasureTrialTask(metric=metric,
trial_results=[trial_result],
state=JobState.DONE,
result=metric_result)
task.save()
self.measure_trial_tasks[
metric_result.identifier] = task.identifier
def setUp(self):
# Remove the collections as the start of the test, so that we're sure it's empty
Task._mongometa.collection.drop()
MetricResult._mongometa.collection.drop()
mock_types.MockMetric._mongometa.collection.drop()
self.metrics = [mock_types.MockMetric() for _ in range(2)]
for metric in self.metrics:
metric.save()
self.measure_trial_tasks = {}
self.metric_results = {}
for metric in self.metrics:
self.measure_trial_tasks[metric.identifier] = []
self.metric_results[metric.identifier] = []
for trial_result in self.trials:
metric_result = mock_types.MockMetricResult(
metric=metric, trial_results=[trial_result], success=True)
metric_result.save()
self.metric_results[metric.identifier].append(
metric_result.identifier)
task = MeasureTrialTask(metric=metric,
trial_results=[trial_result],
state=JobState.DONE,
result=metric_result)
task.save()
self.measure_trial_tasks[metric.identifier].append(
task.identifier)
for trial_result in self.unfinished_trials:
task = MeasureTrialTask(metric=metric,
trial_results=[trial_result],
state=JobState.UNSTARTED)
task.save()
self.measure_trial_tasks[metric.identifier].append(
task.identifier)
def test_removes_incomplete_measure_results(self):
# Make tasks of each type, some unstarted, some running, some complete
unstarted = []
running = []
complete = []
for system in self.systems:
for image_collection in self.image_collections:
trial_result = mock_types.MockTrialResult(
system=system, image_source=image_collection, success=True)
trial_result.save()
for metric in self.metrics:
task = MeasureTrialTask(metric=metric,
trial_results=[trial_result],
state=JobState.UNSTARTED)
task.save()
unstarted.append(task)
task = MeasureTrialTask(metric=metric,
trial_results=[trial_result],
state=JobState.RUNNING,
job_id=10,
node_id='this')
task.save()
running.append(task)
metric_result = mock_types.MockMetricResult(
metric=metric,
trial_results=[trial_result],
success=True)
metric_result.save()
task = MeasureTrialTask(metric=metric,
trial_results=[trial_result],
state=JobState.DONE,
result=metric_result)
task.save()
complete.append(task)
self.assertEqual(
len(complete) + len(running) + len(unstarted),
Task.objects.all().count())
invalidate.invalidate_incomplete_tasks()
self.assertEqual(len(complete), Task.objects.all().count())
for unstarted_task in unstarted:
self.assertEqual(
0,
Task.objects.raw({
'_id': unstarted_task.pk
}).count())
for running_task in running:
self.assertEqual(
0,
Task.objects.raw({
'_id': running_task.pk
}).count())
for complete_task in complete:
self.assertEqual(
1,
Task.objects.raw({
'_id': complete_task.pk
}).count())
# Clean up after ourselves
Task.objects.all().delete()
MetricResult.objects.all().delete()
TrialResult.objects.all().delete()