def test_result_id_gets_id_without_dereferencing(self):
result = InitMonitoredResult(
system=self.system,
image_source=self.image_source,
success=True
)
result.save()
obj = RunSystemTask(
system=self.system,
image_source=self.image_source,
state=JobState.DONE,
result=result
)
obj.save()
# Set up mocks
dereferenced = False
def init_side_effect(_):
nonlocal dereferenced
dereferenced = True
InitMonitoredResult.side_effect = init_side_effect
# Delete and reload the object to reset the references to object ids
del obj
obj = next(Task.objects.all())
# Autoload the model types
_ = obj.result_id
self.assertFalse(dereferenced)
# Clean up
InitMonitoredResult.side_effect = None
def setUp(self):
# Remove the collections as the start of the test, so that we're sure it's empty
Task._mongometa.collection.drop()
TrialResult._mongometa.collection.drop()
MetricResult._mongometa.collection.drop()
# Create run system tasks and trial results
self.run_system_tasks = {}
self.trial_result_groups = []
for image_collection in self.image_collections:
for system in self.systems:
trial_result_group = []
for repeat in range(3):
trial_result = mock_types.MockTrialResult(
system=system,
image_source=image_collection,
success=True)
trial_result.save()
trial_result_group.append(trial_result)
task = RunSystemTask(system=system,
image_source=image_collection,
state=JobState.DONE,
result=trial_result)
task.save()
self.run_system_tasks[
trial_result.identifier] = task.identifier
self.trial_result_groups.append(trial_result_group)
self.measure_trial_tasks = {}
self.metric_results = {}
for group_id, trial_result_group in enumerate(
self.trial_result_groups):
self.measure_trial_tasks[group_id] = []
self.metric_results[group_id] = []
for metric in self.metrics:
metric_result = mock_types.MockMetricResult(
metric=metric,
trial_results=trial_result_group,
success=True)
metric_result.save()
self.metric_results[group_id].append(metric_result.identifier)
task = MeasureTrialTask(metric=metric,
trial_results=trial_result_group,
state=JobState.DONE,
result=metric_result)
task.save()
self.measure_trial_tasks[group_id].append(task.identifier)
for metric in self.unfinished_metrics:
task = MeasureTrialTask(metric=metric,
trial_results=trial_result_group,
state=JobState.UNSTARTED)
task.save()
self.measure_trial_tasks[group_id].append(task.identifier)
def test_stores_and_loads_unstarted(self):
obj = RunSystemTask(
system=self.system,
image_source=self.image_source,
state=JobState.UNSTARTED
)
obj.save()
# Load all the entities
all_entities = list(Task.objects.all())
self.assertGreaterEqual(len(all_entities), 1)
self.assertEqual(all_entities[0], obj)
all_entities[0].delete()
def test_run_task_saves_result(self):
trial_result = mock.create_autospec(mock_types.MockTrialResult)
self.system.finish_trial = lambda: trial_result
subject = RunSystemTask(
system=self.system,
image_source=self.image_source,
state=JobState.RUNNING,
node_id='test',
job_id=1
)
self.assertFalse(trial_result.save.called)
subject.run_task(self.path_manager)
self.assertTrue(trial_result.save.called)
def test_run_task_records_returned_trial_result(self):
trial_result = mock.create_autospec(mock_types.MockTrialResult)
self.system.finish_trial = lambda: trial_result
subject = RunSystemTask(
system=self.system,
image_source=self.image_source,
state=JobState.RUNNING,
node_id='test',
job_id=1
)
self.assertIsNone(subject.result)
subject.run_task(self.path_manager)
self.assertTrue(subject.is_finished)
self.assertEqual(subject.result, trial_result)
def test_result_id_is_none_if_result_is_none(self):
obj = RunSystemTask(
system=self.system,
image_source=self.image_source,
state=JobState.UNSTARTED
)
self.assertIsNone(obj.result_id)
def test_run_task_passes_through_seed(self):
trial_result = mock.create_autospec(mock_types.MockTrialResult)
self.system.finish_trial = lambda: trial_result
seed = 14578
mock_system = mock.create_autospec(VisionSystem, spec_set=True)
subject = RunSystemTask(
system=mock_system,
image_source=self.image_source,
state=JobState.RUNNING,
seed=seed,
node_id='test',
job_id=1
)
subject.run_task(self.path_manager)
self.assertTrue(mock_system.start_trial.called)
self.assertEqual(mock.call(self.image_source.sequence_type, seed=seed), mock_system.start_trial.call_args)
def test_run_task_records_unable_to_measure_trial(self):
self.system.is_image_source_appropriate = lambda _: False
subject = RunSystemTask(
system=self.system,
image_source=self.image_source,
state=JobState.RUNNING,
node_id='test',
job_id=1
)
self.assertIsNone(subject.result)
subject.run_task(self.path_manager)
self.assertTrue(subject.is_finished)
self.assertIsNotNone(subject.result)
self.assertFalse(subject.result.success)
self.assertIsNotNone(subject.result.message)
self.assertEqual(self.system, subject.result.system)
self.assertEqual(self.image_source, subject.result.image_source)
def test_result_id_is_result_primary_key(self):
result = mock_types.MockTrialResult()
result.pk = bson.ObjectId()
obj = RunSystemTask(
system=self.system,
image_source=self.image_source,
state=JobState.DONE,
result=result
)
self.assertEqual(result.pk, obj.result_id)
def setUp(self):
# Remove the collections as the start of the test, so that we're sure it's empty
Task._mongometa.collection.drop()
Image._mongometa.collection.drop()
TrialResult._mongometa.collection.drop()
mock_types.MockSystem._mongometa.collection.drop()
# Create the basic image sources in two groups
self.image_collections = [make_image_collection() for _ in range(3)]
self.unfinisned_image_collections = [
make_image_collection() for _ in range(3)
]
# Make systesm
self.systems = [mock_types.MockSystem() for _ in range(2)]
# Create run system tasks and trial results
self.run_system_tasks = {}
self.trial_results = {}
for system in self.systems:
system.save()
self.run_system_tasks[system.identifier] = []
self.trial_results[system.identifier] = []
for image_collection in self.image_collections:
trial_result = mock_types.MockTrialResult(
system=system, image_source=image_collection, success=True)
trial_result.save()
self.trial_results[system.identifier].append(
trial_result.identifier)
task = RunSystemTask(system=system,
image_source=image_collection,
state=JobState.DONE,
result=trial_result)
task.save()
self.run_system_tasks[system.identifier].append(
task.identifier)
for image_collection in self.unfinisned_image_collections:
task = RunSystemTask(
system=system,
image_source=image_collection,
state=JobState.UNSTARTED,
)
task.save()
self.run_system_tasks[system.identifier].append(
task.identifier)
def test_run_task_records_exception_during_execution_and_re_raises(self):
message = 'No mercy. No respite.'
def bad_measure_results(*_, **__):
raise ValueError(message)
self.system.start_trial = bad_measure_results
subject = RunSystemTask(
system=self.system,
image_source=self.image_source,
state=JobState.RUNNING,
node_id='test',
job_id=1
)
self.assertIsNone(subject.result)
with self.assertRaises(ValueError):
subject.run_task(self.path_manager)
self.assertTrue(subject.is_finished)
self.assertIsNotNone(subject.result)
self.assertFalse(subject.result.success)
self.assertIsNotNone(subject.result.message)
self.assertIn(message, subject.result.message)
self.assertEqual(self.system, subject.result.system)
self.assertEqual(self.image_source, subject.result.image_source)
def test_load_referenced_models_does_nothing_to_models_that_are_already_objects(self, mock_autoload):
# Set up objects
obj = RunSystemTask(
system=self.system,
image_source=self.image_source,
state=JobState.DONE,
result=self.trial_result
)
obj.save()
self.assertFalse(mock_autoload.called)
obj.load_referenced_models()
self.assertFalse(mock_autoload.called)
def test_get_result_autoloads_model_type_before_dereferencing(self, mock_autoload):
# Set up objects
result = InitMonitoredResult(
system=self.system,
image_source=self.image_source,
success=True
)
result.save()
obj = RunSystemTask(
system=self.system,
image_source=self.image_source,
state=JobState.DONE,
result=result
)
obj.save()
# Set up mocks
loaded = False
constructed = False
loaded_first = False
def autoload_side_effect(model, *_, **__):
nonlocal loaded
if model == TrialResult:
loaded = True
mock_autoload.side_effect = autoload_side_effect
def init_result_side_effect(_):
nonlocal loaded, constructed, loaded_first
constructed = True
if loaded:
loaded_first = True
InitMonitoredResult.side_effect = init_result_side_effect
# Delete and reload the object to reset the references to object ids
del obj
obj = next(Task.objects.all())
# get the result
obj.get_result()
self.assertTrue(mock_autoload.called)
self.assertEqual(mock.call(TrialResult, [result.pk]), mock_autoload.call_args)
self.assertTrue(constructed)
self.assertTrue(loaded)
self.assertTrue(loaded_first)
# Clean up
InitMonitoredResult.side_effect = None
def test_load_referenced_models_loads_system_and_image_source_models(self, mock_autoload):
obj = RunSystemTask(
system=self.system,
image_source=self.image_source,
state=JobState.UNSTARTED
)
obj.save()
# Delete and reload the object to reset the references to object ids
del obj
obj = next(Task.objects.all())
# Auto load the model types
self.assertFalse(mock_autoload.called)
obj.load_referenced_modules()
self.assertTrue(mock_autoload.called)
self.assertIn(mock.call(VisionSystem, [self.system.pk]), mock_autoload.call_args_list)
self.assertIn(mock.call(ImageSource, [self.image_source.pk]), mock_autoload.call_args_list)
def test_load_referenced_models_autoloads_models_that_are_just_ids(self, mock_autoload):
# Set up objects
obj = RunSystemTask(
system=self.system,
image_source=self.image_source,
state=JobState.DONE,
result=self.trial_result
)
obj.save()
obj_id = obj.pk
del obj # Clear existing references, which should reset the references to ids
obj = RunSystemTask.objects.get({'_id': obj_id})
self.assertFalse(mock_autoload.called)
obj.load_referenced_models()
self.assertTrue(mock_autoload.called)
self.assertIn(mock.call(VisionSystem, [self.system.pk]), mock_autoload.call_args_list)
self.assertIn(mock.call(ImageSource, [self.image_source.pk]), mock_autoload.call_args_list)
self.assertIn(mock.call(TrialResult, [self.trial_result.pk]), mock_autoload.call_args_list)
def get_run_system_task(system: typing.Union[VisionSystem, bson.ObjectId],
image_source: typing.Union[ImageSource, bson.ObjectId],
repeat: int = 0,
seed: int = 0,
num_cpus: int = 1,
num_gpus: int = 0,
memory_requirements: str = '3GB',
expected_duration: str = '1:00:00') -> RunSystemTask:
"""
Get a task to run a system.
Most of the parameters are resources requirements passed to the job system.
:param system: The id of the vision system to test
:param image_source: The id of the image source to test with
:param repeat: The repeat of this trial, so we can run the same system more than once.
:param seed: The random seed to use. Ignored for anything except SEEDED systems.
:param num_cpus: The number of CPUs required for the job. Default 1.
:param num_gpus: The number of GPUs required for the job. Default 0.
:param memory_requirements: The memory required for this job. Default 3 GB.
:param expected_duration: The expected time this job will take. Default 1 hour.
:return: A RunSystemTask
"""
if isinstance(system, VisionSystem):
use_seed = (system.is_deterministic() is StochasticBehaviour.SEEDED)
use_repeat = (system.is_deterministic()
is not StochasticBehaviour.DETERMINISTIC)
system = system.identifier
else:
if VisionSystem.objects.raw({'_id': system}).count() < 1:
raise ValueError(
f"system \"{system}\" is not a valid VisionSystem id")
system_classes = get_model_classes(VisionSystem, [system])
if len(system_classes) >= 1:
use_seed = (system_classes[0].is_deterministic() is
StochasticBehaviour.SEEDED)
use_repeat = (system_classes[0].is_deterministic()
is not StochasticBehaviour.DETERMINISTIC)
else:
raise ValueError(f"Could not load class for system {system}")
if isinstance(image_source, ImageSource):
image_source = image_source.identifier
elif ImageSource.objects.raw({'_id': image_source}).count() < 1:
raise ValueError(
f"image_source \"{image_source}\" is not a valid ImageSource id")
query = {'system': system, 'image_source': image_source}
if use_repeat:
query['repeat'] = int(repeat)
if use_seed:
query['seed'] = int(seed)
try:
return RunSystemTask.objects.get(query)
except RunSystemTask.DoesNotExist:
return RunSystemTask(system=system,
image_source=image_source,
repeat=int(repeat) if use_repeat else 0,
seed=int(seed) if use_seed else None,
state=JobState.UNSTARTED,
num_cpus=int(num_cpus),
num_gpus=int(num_gpus),
memory_requirements=str(memory_requirements),
expected_duration=str(expected_duration))
def setUp(self):
# Remove the collections as the start of the test, so that we're sure it's empty
Task._mongometa.collection.drop()
Image._mongometa.collection.drop()
ImageCollection._mongometa.collection.drop()
TrialResult._mongometa.collection.drop()
# Create systems in two groups
self.systems = [mock_types.MockSystem() for _ in range(2)]
for system in self.systems:
system.save()
self.unfinished_systems = [mock_types.MockSystem() for _ in range(2)]
for system in self.unfinished_systems:
system.save()
# Make image collections with import tasks
self.image_collections = []
self.import_dataset_tasks = {}
for _ in range(2):
image_collection = make_image_collection()
image_collection.save()
self.image_collections.append(image_collection)
task = ImportDatasetTask(module_name='myloader',
path='test/filename',
result=image_collection,
state=JobState.DONE)
task.save()
self.import_dataset_tasks[
image_collection.identifier] = task.identifier
# Create run system tasks and trial results on each image collection
self.run_system_tasks = {}
self.trial_results = {}
for image_collection in self.image_collections:
self.run_system_tasks[image_collection.identifier] = []
self.trial_results[image_collection.identifier] = []
for system in self.systems:
trial_result = mock_types.MockTrialResult(
system=system, image_source=image_collection, success=True)
trial_result.save()
self.trial_results[image_collection.identifier].append(
trial_result.identifier)
task = RunSystemTask(system=system,
image_source=image_collection,
state=JobState.DONE,
result=trial_result)
task.save()
self.run_system_tasks[image_collection.identifier].append(
task.identifier)
for system in self.unfinished_systems:
task = RunSystemTask(
system=system,
image_source=image_collection,
state=JobState.UNSTARTED,
)
task.save()
self.run_system_tasks[image_collection.identifier].append(
task.identifier)
def test_saving_throws_exeption_if_required_fields_are_missing(self):
obj = RunSystemTask(
# system=self.system,
image_source=self.image_source,
state=JobState.UNSTARTED
)
with self.assertRaises(ValidationError):
obj.save()
obj = RunSystemTask(
system=self.system,
# image_source=self.image_source,
state=JobState.UNSTARTED
)
with self.assertRaises(ValidationError):
obj.save()
obj = RunSystemTask(
system=self.system,
image_source=self.image_source
# state=JobState.UNSTARTED
)
with self.assertRaises(ValidationError):
obj.save()
def test_removes_incomplete_run_system_tasks(self):
# Make tasks of each type, some unstarted, some running, some complete
unstarted = []
running = []
complete = []
for system in self.systems:
for image_collection in self.image_collections:
task = RunSystemTask(system=system,
image_source=image_collection,
state=JobState.UNSTARTED)
task.save()
unstarted.append(task)
task = RunSystemTask(system=system,
image_source=image_collection,
state=JobState.RUNNING,
job_id=10,
node_id='this')
task.save()
running.append(task)
trial_result = mock_types.MockTrialResult(
system=system, image_source=image_collection, success=True)
trial_result.save()
task = RunSystemTask(system=system,
image_source=image_collection,
state=JobState.DONE,
result=trial_result)
task.save()
complete.append(task)
self.assertEqual(
len(complete) + len(running) + len(unstarted),
Task.objects.all().count())
invalidate.invalidate_incomplete_tasks()
self.assertEqual(len(complete), Task.objects.all().count())
for unstarted_task in unstarted:
self.assertEqual(
0,
Task.objects.raw({
'_id': unstarted_task.pk
}).count())
for running_task in running:
self.assertEqual(
0,
Task.objects.raw({
'_id': running_task.pk
}).count())
for complete_task in complete:
self.assertEqual(
1,
Task.objects.raw({
'_id': complete_task.pk
}).count())
# Clean up after ourselves
Task.objects.all().delete()
TrialResult.objects.all().delete()