def test_schedule_tasks_discards_old_metric_results(
self, mock_task_manager, mock_load_minimal_trial):
systems = [mock_core.MockSystem(_id=ObjectId()) for _ in range(3)]
systems[0].is_deterministic = mock.create_autospec(
systems[0].is_deterministic,
return_value=StochasticBehaviour.NON_DETERMINISTIC)
image_sources = [
mock_core.MockImageSource(_id=ObjectId()) for _ in range(3)
]
metrics = [mock_core.MockMetric(_id=ObjectId()) for _ in range(3)]
repeats = 2
trial_results_map = make_trial_map(systems, image_sources, repeats)
make_mock_get_run_system_task(mock_task_manager, trial_results_map)
patch_load_minimal_trial(mock_load_minimal_trial)
old_metric_results = [
mock_core.MockMetricResult(_id=ObjectId(), success=True)
]
metric_result_ids = []
def mock_get_measure_trial_task(*_, **__):
result_id = ObjectId()
metric_result_ids.append(result_id)
mock_task = mock.MagicMock()
mock_task.is_finished = True
mock_task.result_id = result_id
return mock_task
mock_task_manager.get_measure_trial_task.side_effect = mock_get_measure_trial_task
subject = SimpleExperiment(name="TestSimpleExperiment",
systems=systems,
image_sources=image_sources,
metrics=metrics,
repeats=repeats,
metric_results=old_metric_results)
subject.schedule_tasks()
with no_auto_dereference(SimpleExperiment):
# Results should be all ids, from the measure_all
for metric_result_id in metric_result_ids:
self.assertIn(metric_result_id, subject.metric_results)
for metric_result in old_metric_results:
self.assertNotIn(metric_result.pk, subject.metric_results)
def test_schedule_tasks_runs_all_systems_with_all_image_sources_repeatedly(
self, mock_task_manager):
systems = [mock_core.MockSystem(_id=ObjectId()) for _ in range(3)]
systems[0].is_deterministic = mock.create_autospec(
systems[0].is_deterministic,
return_value=StochasticBehaviour.NON_DETERMINISTIC)
image_sources = [
mock_core.MockImageSource(_id=ObjectId()) for _ in range(3)
]
metric = mock_core.MockMetric(_id=ObjectId())
repeats = 2
run_cpus = 3
run_gpus = 2
run_memory = '9GB'
run_duration = '2:33:44'
subject = SimpleExperiment(name="TestSimpleExperiment",
systems=systems,
image_sources=image_sources,
metrics=[metric],
repeats=repeats,
run_cpus=run_cpus,
run_gpus=run_gpus,
run_memory=run_memory,
run_duration=run_duration)
subject.schedule_tasks()
for system in systems:
for image_source in image_sources:
actual_repeats = repeats if system.is_deterministic(
) is not StochasticBehaviour.DETERMINISTIC else 1
for repeat in range(actual_repeats):
self.assertIn(
mock.call(system=system,
image_source=image_source,
repeat=repeat,
num_cpus=run_cpus,
num_gpus=run_gpus,
memory_requirements=run_memory,
expected_duration=run_duration),
mock_task_manager.get_run_system_task.call_args_list)
def test_schedule_tasks_uses_run_system_tasks_correctly(
self, mock_task_manager, mock_load_minimal_trial):
systems = [mock_core.MockSystem(_id=ObjectId()) for _ in range(1)]
image_sources = [
mock_core.MockImageSource(_id=ObjectId()) for _ in range(1)
]
metrics = [mock_core.MockMetric(_id=ObjectId()) for _ in range(1)]
repeats = 1
trial_results_map = make_trial_map(systems, image_sources, repeats)
# Note: This is a separate test because autospecs are expensive in a loop
make_mock_get_run_system_task(mock_task_manager,
trial_results_map,
autospec=True)
patch_load_minimal_trial(mock_load_minimal_trial)
subject = SimpleExperiment(name="TestSimpleExperiment",
systems=systems,
image_sources=image_sources,
metrics=metrics,
repeats=repeats)
subject.schedule_tasks()
def test_schedule_tasks_uses_measure_trial_tasks_correctly(
self, mock_task_manager, mock_load_minimal_trial):
systems = [mock_core.MockSystem(_id=ObjectId()) for _ in range(1)]
image_sources = [
mock_core.MockImageSource(_id=ObjectId()) for _ in range(1)
]
metrics = [mock_core.MockMetric(_id=ObjectId()) for _ in range(1)]
repeats = 1
trial_results_map = make_trial_map(systems, image_sources, repeats)
make_mock_get_run_system_task(mock_task_manager, trial_results_map)
patch_load_minimal_trial(mock_load_minimal_trial)
metric_result_ids = []
def mock_get_measure_trial_task(*_, **__):
result_id = ObjectId()
metric_result_ids.append(result_id)
mock_task = mock.create_autospec(MeasureTrialTask)
mock_task.is_finished = True
mock_task.result_id = result_id
return mock_task
mock_task_manager.get_measure_trial_task.side_effect = mock_get_measure_trial_task
subject = SimpleExperiment(
name="TestSimpleExperiment",
systems=systems,
image_sources=image_sources,
metrics=metrics,
repeats=repeats,
)
subject.schedule_tasks()
with no_auto_dereference(SimpleExperiment):
for metric_result_id in metric_result_ids:
self.assertIn(metric_result_id, subject.metric_results)
def test_schedule_tasks_runs_all_systems_with_all_image_sources_with_all_seeds(
self, mock_task_manager):
systems = [mock_core.MockSystem(_id=ObjectId()) for _ in range(3)]
systems[0].is_deterministic = mock.create_autospec(
systems[0].is_deterministic,
return_value=StochasticBehaviour.SEEDED)
image_sources = [
mock_core.MockImageSource(_id=ObjectId()) for _ in range(3)
]
metric = mock_core.MockMetric(_id=ObjectId())
repeats = 2
subject = SimpleExperiment(name="TestSimpleExperiment",
systems=systems,
image_sources=image_sources,
metrics=[metric],
repeats=repeats,
use_seed=True)
subject.schedule_tasks()
self.assertEqual(repeats, len(
subject.seeds)) # The set of seeds should have auto-filled
self.assertEqual(len(subject.seeds), len(set(
subject.seeds))) # The set of seeds must be distinct
for system in systems:
for image_source in image_sources:
for seed in subject.seeds:
self.assertIn(
mock.call(system=system,
image_source=image_source,
repeat=0,
seed=seed,
num_cpus=mock.ANY,
num_gpus=mock.ANY,
memory_requirements=mock.ANY,
expected_duration=mock.ANY),
mock_task_manager.get_run_system_task.call_args_list)
def test_schedule_tasks_measures_all_trial_results(
self, mock_task_manager, mock_load_minimal_trial):
systems = [mock_core.MockSystem(_id=ObjectId()) for _ in range(3)]
systems[0].is_deterministic = mock.create_autospec(
systems[0].is_deterministic,
return_value=StochasticBehaviour.NON_DETERMINISTIC)
image_sources = [
mock_core.MockImageSource(_id=ObjectId()) for _ in range(3)
]
metrics = [mock_core.MockMetric(_id=ObjectId()) for _ in range(3)]
repeats = 3
measure_cpus = 16
measure_gpus = 22
measure_memory = '88GB'
measure_duration = '0:01:01'
trial_results_map = make_trial_map(systems, image_sources, repeats)
trial_groups = [
trial_group for trials_by_source in trial_results_map.values()
for trial_group in trials_by_source.values()
]
trials_by_id = {
trial_result.pk: trial_result
for trial_group in trial_groups for trial_result in trial_group
}
make_mock_get_run_system_task(mock_task_manager,
trial_results_map,
autospec=False)
patch_load_minimal_trial(mock_load_minimal_trial)
def mock_load_minimal_trial_impl(object_id):
if object_id in trials_by_id:
return trials_by_id[object_id]
raise ValueError(f"Unidentified trial result id {object_id}")
mock_load_minimal_trial.side_effect = mock_load_minimal_trial_impl
subject = SimpleExperiment(name="TestSimpleExperiment",
systems=systems,
image_sources=image_sources,
metrics=metrics,
repeats=repeats,
measure_cpus=measure_cpus,
measure_gpus=measure_gpus,
measure_memory=measure_memory,
measure_duration=measure_duration)
subject.schedule_tasks()
for group_idx, trial_group in enumerate(trial_groups):
for metric_idx, metric in enumerate(metrics):
self.assertIn(
mock.call(trial_results=trial_group,
metric=metric,
num_cpus=measure_cpus,
num_gpus=measure_gpus,
memory_requirements=measure_memory,
expected_duration=measure_duration),
mock_task_manager.get_measure_trial_task.call_args_list,
"\nCouldn't find measure_trial_task for metric {0}, trial group {1}"
.format(metric_idx, group_idx))