def test_returns_different_error_messages_if_systems_or_image_sources_are_different(self):
system1 = mock_types.MockSystem()
system2 = mock_types.MockSystem()
image_source1 = mock_types.MockImageSource()
image_source2 = mock_types.MockImageSource()
group1 = [
mock_types.MockTrialResult(image_source=image_source1, system=system1, success=True)
for _ in range(10)
]
group2 = [
mock_types.MockTrialResult(image_source=image_source2, system=system1, success=True)
for _ in range(10)
]
group3 = [
mock_types.MockTrialResult(image_source=image_source1, system=system2, success=True)
for _ in range(10)
]
different_image_source_msg = mtr.check_trial_collection(group1 + group2)
self.assertIn('image source', different_image_source_msg)
different_system_msg = mtr.check_trial_collection(group1 + group3)
self.assertNotEqual(different_system_msg, different_image_source_msg)
self.assertIn('system', different_system_msg)
def test_add_vision_systems_enforces_uniqueness_if_already_dereferenced(
self):
system1 = mock_core.MockSystem()
system1.save()
system2 = mock_core.MockSystem()
system2.save()
obj = SimpleExperiment(name="TestSimpleExperiment", systems=[system1])
obj.add_vision_systems([system1, system2])
self.assertEqual(obj.systems, [system1, system2])
# check we can still save
obj.save()
def test_returns_error_message_if_trials_have_different_systems(self):
system1 = mock_types.MockSystem()
system2 = mock_types.MockSystem()
image_source = mock_types.MockImageSource()
group1 = [
mock_types.MockTrialResult(image_source=image_source, system=system1, success=True)
for _ in range(10)
]
group2 = [
mock_types.MockTrialResult(image_source=image_source, system=system2, success=True)
for _ in range(10)
]
message = mtr.check_trial_collection(group1 + group2)
self.assertIsNotNone(message)
self.assertIn('system', message)
def test_load_referenced_models_autoloads_models_that_are_just_ids(
self, mock_autoload):
system = mock_core.MockSystem()
system.save()
image_source = mock_core.MockImageSource()
image_source.save()
metric = mock_core.MockMetric()
metric.save()
obj = SimpleExperiment(name="TestSimpleExperiment",
systems=[system],
image_sources=[image_source],
metrics=[metric])
obj.save()
del obj # Clear existing references, which should reset the references to ids
obj = next(SimpleExperiment.objects.all())
self.assertFalse(mock_autoload.called)
obj.load_referenced_models()
self.assertTrue(mock_autoload.called)
self.assertIn(mock.call(VisionSystem, ids=[system.pk]),
mock_autoload.call_args_list)
self.assertIn(mock.call(ImageSource, ids=[image_source.pk]),
mock_autoload.call_args_list)
self.assertIn(mock.call(Metric, ids=[metric.pk]),
mock_autoload.call_args_list)
def setUp(self):
system = mock_types.MockSystem()
image_source = mock_types.MockImageSource()
self.path_manager = PathManager(['~'], '~/tmp')
self.metric = mock_types.MockTrialComparisonMetric()
self.trial_result_1 = mock_types.MockTrialResult(system=system, image_source=image_source, success=False)
self.trial_result_2 = mock_types.MockTrialResult(system=system, image_source=image_source, success=False)
def setUpClass(cls):
cls.system = mock_types.MockSystem()
cls.image_source = mock_types.MockImageSource()
image_data = [
np.random.normal(128, 20, size=(10, 10)).astype(np.uint8)
for _ in range(cls.num_images)
]
cls.images = [
Image(
pixels=pixels,
metadata=make_metadata(
pixels=pixels,
source_type=ImageSourceType.SYNTHETIC,
camera_pose=Transform(
(idx * 15, idx, 0),
tf3d.quaternions.axangle2quat((1, 2, 3), 5 * idx * np.pi / (2 * cls.num_images)), w_first=True
),
intrinsics=CameraIntrinsics(
width=10, height=10,
fx=5, fy=5,
cx=5, cy=5
)
)
)
for idx, pixels in enumerate(image_data)
]
def test_returns_none_if_all_trials_succeed_and_have_same_image_source_and_system(self):
system = mock_types.MockSystem()
image_source = mock_types.MockImageSource()
group1 = [
mock_types.MockTrialResult(image_source=image_source, system=system, success=True)
for _ in range(10)
]
self.assertIsNone(mtr.check_trial_collection(group1))
def setUpClass(cls):
dbconn.connect_to_test_db()
cls.system = mock_types.MockSystem()
cls.image_source = mock_types.MockImageSource()
cls.system.save()
cls.image_source.save()
cls.trial_result = mock_types.MockTrialResult(image_source=cls.image_source, system=cls.system, success=True)
cls.trial_result.save()
def setUpClass(cls):
dbconn.connect_to_test_db()
dbconn.setup_image_manager()
cls.path_manager = PathManager(['~'], '~/tmp')
cls.system = mock_types.MockSystem()
cls.image_source = mock_types.MockImageSource()
cls.image_source.build_images()
cls.system.save()
cls.image_source.save()
def setUpClass(cls):
dbconn.connect_to_test_db()
dbconn.setup_image_manager()
# Create the basic image sources, systems, and metrics.
cls.image_collections = [make_image_collection() for _ in range(2)]
cls.systems = [mock_types.MockSystem() for _ in range(2)]
for system in cls.systems:
system.save()
cls.metrics = [mock_types.MockMetric() for _ in range(2)]
for metric in cls.metrics:
metric.save()
def setUpClass(cls):
dbconn.connect_to_test_db()
cls.system = mock_types.MockSystem()
cls.image_source = mock_types.MockImageSource()
cls.metric = mock_types.MockTrialComparisonMetric()
cls.system.save()
cls.image_source.save()
cls.metric.save()
cls.trial_result_1 = mock_types.MockTrialResult(
image_source=cls.image_source, system=cls.system, success=True)
cls.trial_result_1.save()
cls.trial_result_2 = mock_types.MockTrialResult(
image_source=cls.image_source, system=cls.system, success=True)
cls.trial_result_2.save()
def setUp(self):
# Remove the collections as the start of the test, so that we're sure it's empty
Task._mongometa.collection.drop()
Image._mongometa.collection.drop()
TrialResult._mongometa.collection.drop()
mock_types.MockSystem._mongometa.collection.drop()
# Create the basic image sources in two groups
self.image_collections = [make_image_collection() for _ in range(3)]
self.unfinisned_image_collections = [
make_image_collection() for _ in range(3)
]
# Make systesm
self.systems = [mock_types.MockSystem() for _ in range(2)]
# Create run system tasks and trial results
self.run_system_tasks = {}
self.trial_results = {}
for system in self.systems:
system.save()
self.run_system_tasks[system.identifier] = []
self.trial_results[system.identifier] = []
for image_collection in self.image_collections:
trial_result = mock_types.MockTrialResult(
system=system, image_source=image_collection, success=True)
trial_result.save()
self.trial_results[system.identifier].append(
trial_result.identifier)
task = RunSystemTask(system=system,
image_source=image_collection,
state=JobState.DONE,
result=trial_result)
task.save()
self.run_system_tasks[system.identifier].append(
task.identifier)
for image_collection in self.unfinisned_image_collections:
task = RunSystemTask(
system=system,
image_source=image_collection,
state=JobState.UNSTARTED,
)
task.save()
self.run_system_tasks[system.identifier].append(
task.identifier)
def test_schedule_tasks_discards_old_metric_results(
self, mock_task_manager, mock_load_minimal_trial):
systems = [mock_core.MockSystem(_id=ObjectId()) for _ in range(3)]
systems[0].is_deterministic = mock.create_autospec(
systems[0].is_deterministic,
return_value=StochasticBehaviour.NON_DETERMINISTIC)
image_sources = [
mock_core.MockImageSource(_id=ObjectId()) for _ in range(3)
]
metrics = [mock_core.MockMetric(_id=ObjectId()) for _ in range(3)]
repeats = 2
trial_results_map = make_trial_map(systems, image_sources, repeats)
make_mock_get_run_system_task(mock_task_manager, trial_results_map)
patch_load_minimal_trial(mock_load_minimal_trial)
old_metric_results = [
mock_core.MockMetricResult(_id=ObjectId(), success=True)
]
metric_result_ids = []
def mock_get_measure_trial_task(*_, **__):
result_id = ObjectId()
metric_result_ids.append(result_id)
mock_task = mock.MagicMock()
mock_task.is_finished = True
mock_task.result_id = result_id
return mock_task
mock_task_manager.get_measure_trial_task.side_effect = mock_get_measure_trial_task
subject = SimpleExperiment(name="TestSimpleExperiment",
systems=systems,
image_sources=image_sources,
metrics=metrics,
repeats=repeats,
metric_results=old_metric_results)
subject.schedule_tasks()
with no_auto_dereference(SimpleExperiment):
# Results should be all ids, from the measure_all
for metric_result_id in metric_result_ids:
self.assertIn(metric_result_id, subject.metric_results)
for metric_result in old_metric_results:
self.assertNotIn(metric_result.pk, subject.metric_results)
def test_schedule_tasks_runs_all_systems_with_all_image_sources_repeatedly(
self, mock_task_manager):
systems = [mock_core.MockSystem(_id=ObjectId()) for _ in range(3)]
systems[0].is_deterministic = mock.create_autospec(
systems[0].is_deterministic,
return_value=StochasticBehaviour.NON_DETERMINISTIC)
image_sources = [
mock_core.MockImageSource(_id=ObjectId()) for _ in range(3)
]
metric = mock_core.MockMetric(_id=ObjectId())
repeats = 2
run_cpus = 3
run_gpus = 2
run_memory = '9GB'
run_duration = '2:33:44'
subject = SimpleExperiment(name="TestSimpleExperiment",
systems=systems,
image_sources=image_sources,
metrics=[metric],
repeats=repeats,
run_cpus=run_cpus,
run_gpus=run_gpus,
run_memory=run_memory,
run_duration=run_duration)
subject.schedule_tasks()
for system in systems:
for image_source in image_sources:
actual_repeats = repeats if system.is_deterministic(
) is not StochasticBehaviour.DETERMINISTIC else 1
for repeat in range(actual_repeats):
self.assertIn(
mock.call(system=system,
image_source=image_source,
repeat=repeat,
num_cpus=run_cpus,
num_gpus=run_gpus,
memory_requirements=run_memory,
expected_duration=run_duration),
mock_task_manager.get_run_system_task.call_args_list)
def setUpClass(cls):
dbconn.connect_to_test_db()
dbconn.setup_image_manager()
# Create the basic image sources, systems, and metrics.
cls.image_collections = [make_image_collection() for _ in range(2)]
cls.systems = [mock_types.MockSystem() for _ in range(2)]
for system in cls.systems:
system.save()
cls.metrics = [mock_types.MockMetric() for _ in range(2)]
for metric in cls.metrics:
metric.save()
for image_collection in cls.image_collections:
for system in cls.systems:
trial_result = mock_types.MockTrialResult(
system=system, image_source=image_collection, success=True)
trial_result.save()
cls.trials.append(trial_result)
def test_load_referenced_models_does_nothing_to_models_that_are_already_objects(
self, mock_autoload):
system = mock_core.MockSystem()
system.save()
image_source = mock_core.MockImageSource()
image_source.save()
metric = mock_core.MockMetric()
metric.save()
obj = SimpleExperiment(name="TestSimpleExperiment",
systems=[system],
image_sources=[image_source],
metrics=[metric])
obj.save()
self.assertFalse(mock_autoload.called)
obj.load_referenced_models()
self.assertFalse(mock_autoload.called)
def test_schedule_tasks_uses_run_system_tasks_correctly(
self, mock_task_manager, mock_load_minimal_trial):
systems = [mock_core.MockSystem(_id=ObjectId()) for _ in range(1)]
image_sources = [
mock_core.MockImageSource(_id=ObjectId()) for _ in range(1)
]
metrics = [mock_core.MockMetric(_id=ObjectId()) for _ in range(1)]
repeats = 1
trial_results_map = make_trial_map(systems, image_sources, repeats)
# Note: This is a separate test because autospecs are expensive in a loop
make_mock_get_run_system_task(mock_task_manager,
trial_results_map,
autospec=True)
patch_load_minimal_trial(mock_load_minimal_trial)
subject = SimpleExperiment(name="TestSimpleExperiment",
systems=systems,
image_sources=image_sources,
metrics=metrics,
repeats=repeats)
subject.schedule_tasks()
def test_stores_and_loads(self):
system = mock_core.MockSystem()
system.save()
image_source = mock_core.MockImageSource()
image_source.save()
metric = mock_core.MockMetric()
metric.save()
obj = SimpleExperiment(name="TestSimpleExperiment",
enabled=True,
systems=[system],
image_sources=[image_source],
metrics=[metric],
plots=[])
obj.save()
# Load all the entities
all_entities = list(SimpleExperiment.objects.all())
self.assertGreaterEqual(len(all_entities), 1)
self.assertEqual(all_entities[0], obj)
def test_schedule_tasks_runs_all_systems_with_all_image_sources_with_all_seeds(
self, mock_task_manager):
systems = [mock_core.MockSystem(_id=ObjectId()) for _ in range(3)]
systems[0].is_deterministic = mock.create_autospec(
systems[0].is_deterministic,
return_value=StochasticBehaviour.SEEDED)
image_sources = [
mock_core.MockImageSource(_id=ObjectId()) for _ in range(3)
]
metric = mock_core.MockMetric(_id=ObjectId())
repeats = 2
subject = SimpleExperiment(name="TestSimpleExperiment",
systems=systems,
image_sources=image_sources,
metrics=[metric],
repeats=repeats,
use_seed=True)
subject.schedule_tasks()
self.assertEqual(repeats, len(
subject.seeds)) # The set of seeds should have auto-filled
self.assertEqual(len(subject.seeds), len(set(
subject.seeds))) # The set of seeds must be distinct
for system in systems:
for image_source in image_sources:
for seed in subject.seeds:
self.assertIn(
mock.call(system=system,
image_source=image_source,
repeat=0,
seed=seed,
num_cpus=mock.ANY,
num_gpus=mock.ANY,
memory_requirements=mock.ANY,
expected_duration=mock.ANY),
mock_task_manager.get_run_system_task.call_args_list)
def test_schedule_tasks_uses_measure_trial_tasks_correctly(
self, mock_task_manager, mock_load_minimal_trial):
systems = [mock_core.MockSystem(_id=ObjectId()) for _ in range(1)]
image_sources = [
mock_core.MockImageSource(_id=ObjectId()) for _ in range(1)
]
metrics = [mock_core.MockMetric(_id=ObjectId()) for _ in range(1)]
repeats = 1
trial_results_map = make_trial_map(systems, image_sources, repeats)
make_mock_get_run_system_task(mock_task_manager, trial_results_map)
patch_load_minimal_trial(mock_load_minimal_trial)
metric_result_ids = []
def mock_get_measure_trial_task(*_, **__):
result_id = ObjectId()
metric_result_ids.append(result_id)
mock_task = mock.create_autospec(MeasureTrialTask)
mock_task.is_finished = True
mock_task.result_id = result_id
return mock_task
mock_task_manager.get_measure_trial_task.side_effect = mock_get_measure_trial_task
subject = SimpleExperiment(
name="TestSimpleExperiment",
systems=systems,
image_sources=image_sources,
metrics=metrics,
repeats=repeats,
)
subject.schedule_tasks()
with no_auto_dereference(SimpleExperiment):
for metric_result_id in metric_result_ids:
self.assertIn(metric_result_id, subject.metric_results)
def setUpClass(cls):
dbconn.connect_to_test_db()
cls.system = mock_types.MockSystem()
cls.image_source = mock_types.MockImageSource()
cls.system.save()
cls.image_source.save()
def test_schedule_tasks_measures_all_trial_results(
self, mock_task_manager, mock_load_minimal_trial):
systems = [mock_core.MockSystem(_id=ObjectId()) for _ in range(3)]
systems[0].is_deterministic = mock.create_autospec(
systems[0].is_deterministic,
return_value=StochasticBehaviour.NON_DETERMINISTIC)
image_sources = [
mock_core.MockImageSource(_id=ObjectId()) for _ in range(3)
]
metrics = [mock_core.MockMetric(_id=ObjectId()) for _ in range(3)]
repeats = 3
measure_cpus = 16
measure_gpus = 22
measure_memory = '88GB'
measure_duration = '0:01:01'
trial_results_map = make_trial_map(systems, image_sources, repeats)
trial_groups = [
trial_group for trials_by_source in trial_results_map.values()
for trial_group in trials_by_source.values()
]
trials_by_id = {
trial_result.pk: trial_result
for trial_group in trial_groups for trial_result in trial_group
}
make_mock_get_run_system_task(mock_task_manager,
trial_results_map,
autospec=False)
patch_load_minimal_trial(mock_load_minimal_trial)
def mock_load_minimal_trial_impl(object_id):
if object_id in trials_by_id:
return trials_by_id[object_id]
raise ValueError(f"Unidentified trial result id {object_id}")
mock_load_minimal_trial.side_effect = mock_load_minimal_trial_impl
subject = SimpleExperiment(name="TestSimpleExperiment",
systems=systems,
image_sources=image_sources,
metrics=metrics,
repeats=repeats,
measure_cpus=measure_cpus,
measure_gpus=measure_gpus,
measure_memory=measure_memory,
measure_duration=measure_duration)
subject.schedule_tasks()
for group_idx, trial_group in enumerate(trial_groups):
for metric_idx, metric in enumerate(metrics):
self.assertIn(
mock.call(trial_results=trial_group,
metric=metric,
num_cpus=measure_cpus,
num_gpus=measure_gpus,
memory_requirements=measure_memory,
expected_duration=measure_duration),
mock_task_manager.get_measure_trial_task.call_args_list,
"\nCouldn't find measure_trial_task for metric {0}, trial group {1}"
.format(metric_idx, group_idx))
def setUp(self):
self.system = mock_types.MockSystem()
self.image_source = mock_types.MockImageSource()
self.image_source.build_images(10)
self.path_manager = PathManager(['~'], '~/tmp')
def setUp(self):
# Remove the collections as the start of the test, so that we're sure it's empty
Task._mongometa.collection.drop()
Image._mongometa.collection.drop()
ImageCollection._mongometa.collection.drop()
TrialResult._mongometa.collection.drop()
# Create systems in two groups
self.systems = [mock_types.MockSystem() for _ in range(2)]
for system in self.systems:
system.save()
self.unfinished_systems = [mock_types.MockSystem() for _ in range(2)]
for system in self.unfinished_systems:
system.save()
# Make image collections with import tasks
self.image_collections = []
self.import_dataset_tasks = {}
for _ in range(2):
image_collection = make_image_collection()
image_collection.save()
self.image_collections.append(image_collection)
task = ImportDatasetTask(module_name='myloader',
path='test/filename',
result=image_collection,
state=JobState.DONE)
task.save()
self.import_dataset_tasks[
image_collection.identifier] = task.identifier
# Create run system tasks and trial results on each image collection
self.run_system_tasks = {}
self.trial_results = {}
for image_collection in self.image_collections:
self.run_system_tasks[image_collection.identifier] = []
self.trial_results[image_collection.identifier] = []
for system in self.systems:
trial_result = mock_types.MockTrialResult(
system=system, image_source=image_collection, success=True)
trial_result.save()
self.trial_results[image_collection.identifier].append(
trial_result.identifier)
task = RunSystemTask(system=system,
image_source=image_collection,
state=JobState.DONE,
result=trial_result)
task.save()
self.run_system_tasks[image_collection.identifier].append(
task.identifier)
for system in self.unfinished_systems:
task = RunSystemTask(
system=system,
image_source=image_collection,
state=JobState.UNSTARTED,
)
task.save()
self.run_system_tasks[image_collection.identifier].append(
task.identifier)
