def test_push_queued_jobs_runs_qsub_with_each_job_file( self, mock_subprocess, mock_time): mock_time.time = mock.MagicMock() mock_time.time.return_value = 123456789 trainer_id = oid.ObjectId() trainee_id = oid.ObjectId() system_id = oid.ObjectId() image_source_id = oid.ObjectId() trial_id = oid.ObjectId() benchmark_id = oid.ObjectId() subject = hpc.HPCJobSystem({}) with mock.patch('batch_analysis.job_systems.hpc_job_system.open', mock.mock_open(), create=True): subject.queue_train_system(trainer_id, trainee_id) subject.queue_run_system(system_id, image_source_id) subject.queue_benchmark_result(trial_id, benchmark_id) self.assertFalse(mock_subprocess.call.called) subject.run_queued_jobs() self.assertEqual(3, mock_subprocess.call.call_count) self.assertIn( mock.call(['qsub', os.path.expanduser("~/train-123456789.sub")]), mock_subprocess.call.call_args_list) self.assertIn( mock.call(['qsub', os.path.expanduser("~/run-123456789.sub")]), mock_subprocess.call.call_args_list) self.assertIn( mock.call( ['qsub', os.path.expanduser("~/benchmark-123456789.sub")]), mock_subprocess.call.call_args_list)
def test_queue_benchmark_result_returns_true(self): mock_open = mock.mock_open() subject = hpc.HPCJobSystem({}) with mock.patch('batch_analysis.job_systems.hpc_job_system.open', mock_open, create=True): self.assertTrue( subject.queue_benchmark_result(oid.ObjectId(), oid.ObjectId()))
def test_queue_import_dataset_returns_true(self): mock_open = mock.mock_open() subject = hpc.HPCJobSystem({}) with mock.patch('batch_analysis.job_systems.hpc_job_system.open', mock_open, create=True): self.assertTrue( subject.queue_import_dataset('dataset.importer', '/tmp/dataset'))
def test_queue_benchmark_result_uses_configured_environment(self): mock_open = mock.mock_open() virtualenv_path = '/home/user/virtualenv/benchmark-framework/bin/activate' subject = hpc.HPCJobSystem({'environment': virtualenv_path}) with mock.patch('batch_analysis.job_systems.hpc_job_system.open', mock_open, create=True): subject.queue_benchmark_result(oid.ObjectId(), oid.ObjectId()) mock_file = mock_open() self.assertTrue(mock_file.write.called) script_contents = mock_file.write.call_args[0][0] self.assertIn('source {0}'.format(virtualenv_path), script_contents)
def test_queue_run_system_creates_job_file_in_configured_directory(self): mock_open = mock.mock_open() mock_open.return_value = mock.MagicMock() target_folder = os.path.join('/tmp', 'trial-{}'.format(oid.ObjectId())) subject = hpc.HPCJobSystem({'job_location': target_folder}) with mock.patch('batch_analysis.job_systems.hpc_job_system.open', mock_open, create=True): subject.queue_run_system(oid.ObjectId(), oid.ObjectId()) self.assertTrue(mock_open.called) filename = mock_open.call_args[0][0] # Creates in the home directory by default self.assertTrue(filename.startswith(target_folder), "{0} is not in the target directory".format(filename))
def test_queue_import_dataset_indicates_expected_run_time(self): mock_open = mock.mock_open() subject = hpc.HPCJobSystem({}) with mock.patch('batch_analysis.job_systems.hpc_job_system.open', mock_open, create=True): subject.queue_import_dataset('dataset.importer', '/tmp/dataset', expected_duration='125:23:16') self.assertTrue(mock_open.called) mock_file = mock_open() self.assertTrue(mock_file.write.called) script_contents = mock_file.write.call_args[0][0] self.assertIn("#PBS -l walltime={time}".format(time='125:23:16'), script_contents)
def test_queue_run_system_uses_virtualenv_from_environment(self): virtualenv_path = '/home/user/virtualenv/benchmark-framework' mock_open = mock.mock_open() with mock.patch('batch_analysis.job_systems.hpc_job_system.os.environ', {'VIRTUAL_ENV': virtualenv_path}): subject = hpc.HPCJobSystem({}) with mock.patch('batch_analysis.job_systems.hpc_job_system.open', mock_open, create=True): subject.queue_run_system(oid.ObjectId(), oid.ObjectId()) mock_file = mock_open() self.assertTrue(mock_file.write.called) script_contents = mock_file.write.call_args[0][0] self.assertIn('source {0}/bin/activate'.format(virtualenv_path), script_contents)
def test_queue_import_dataset_indicates_desired_memory(self): mock_open = mock.mock_open() subject = hpc.HPCJobSystem({}) with mock.patch('batch_analysis.job_systems.hpc_job_system.open', mock_open, create=True): subject.queue_import_dataset('dataset.importer', '/tmp/dataset', memory_requirements='1542GB') self.assertTrue(mock_open.called) mock_file = mock_open() self.assertTrue(mock_file.write.called) script_contents = mock_file.write.call_args[0][0] self.assertIn("#PBS -l mem={mem}".format(mem='1542GB'), script_contents)
def test_queue_import_dataset_indicates_desired_gpus(self): mock_open = mock.mock_open() subject = hpc.HPCJobSystem({}) with mock.patch('batch_analysis.job_systems.hpc_job_system.open', mock_open, create=True): subject.queue_import_dataset('dataset.importer', '/tmp/dataset', num_gpus=8026) self.assertTrue(mock_open.called) mock_file = mock_open() self.assertTrue(mock_file.write.called) script_contents = mock_file.write.call_args[0][0] self.assertIn("#PBS -l ngpus={gpus}".format(gpus=8026), script_contents) self.assertIn("#PBS -l gputype=M40", script_contents) self.assertIn("#PBS -l cputype=E5-2680v4", script_contents)
def test_queue_import_dataset_passes_experiment_to_task(self): mock_open = mock.mock_open() experiment_id = oid.ObjectId() subject = hpc.HPCJobSystem({}) with mock.patch('batch_analysis.job_systems.hpc_job_system.open', mock_open, create=True): subject.queue_import_dataset('dataset.importer', '/tmp/dataset', experiment_id) mock_file = mock_open() self.assertTrue(mock_file.write.called) script_contents = mock_file.write.call_args[0][0] self.assertIn( "python {0} {1} {2} {3}".format( hpc.quote(task_import_dataset.__file__), 'dataset.importer', '/tmp/dataset', str(experiment_id)), script_contents)
def test_queue_benchmark_result_creates_job_file(self): mock_open = mock.mock_open() mock_open.return_value = mock.MagicMock() subject = hpc.HPCJobSystem({}) trial_id = oid.ObjectId() benchmark_id = oid.ObjectId() with mock.patch('batch_analysis.job_systems.hpc_job_system.open', mock_open, create=True): subject.queue_benchmark_result(trial_id, benchmark_id) self.assertTrue(mock_open.called) filename = mock_open.call_args[0][0] # Creates in the home directory by default self.assertTrue(filename.startswith(os.path.expanduser('~')), "{0} is not in the home directory".format(filename)) self.assertTrue(filename.endswith('.sub'), "{0} does not end with '.sub'".format(filename))
def test_queue_import_dataset_writes_job_script(self): mock_open = mock.mock_open() subject = hpc.HPCJobSystem({}) with mock.patch('batch_analysis.job_systems.hpc_job_system.open', mock_open, create=True): subject.queue_import_dataset('dataset.importer', '/tmp/dataset') self.assertTrue(mock_open.called) mock_file = mock_open() self.assertTrue(mock_file.write.called) script_contents = mock_file.write.call_args[0][0] self.assertTrue(script_contents.startswith('#!/bin/bash'), "Did not create a bash script") self.assertIn( "python {0} {1} {2}".format( hpc.quote(task_import_dataset.__file__), 'dataset.importer', '/tmp/dataset'), script_contents)
def test_queue_import_dataset_creates_job_file(self): mock_open = mock.mock_open() mock_open.return_value = mock.MagicMock() subject = hpc.HPCJobSystem({}) with mock.patch('batch_analysis.job_systems.hpc_job_system.open', mock_open, create=True): subject.queue_import_dataset('dataset.importer', '/tmp/dataset') self.assertTrue(mock_open.called) filename = mock_open.call_args[0][0] # Creates in the home directory by default self.assertTrue(filename.startswith(os.path.expanduser('~')), "{0} is not in the home directory".format(filename)) self.assertNotIn('/tmp/dataset', filename) # we shouldn't add arbitrary slashes self.assertTrue(filename.endswith('.sub'), "{0} does not end with '.sub'".format(filename))
def test_queue_import_dataset_job_name_has_configured_prefix(self): mock_open = mock.mock_open() subject = hpc.HPCJobSystem({'job_name_prefix': 'job_'}) with mock.patch('batch_analysis.job_systems.hpc_job_system.open', mock_open, create=True): subject.queue_import_dataset('dataset.importer', '/tmp/dataset') self.assertTrue(mock_open.called) filename = mock_open.call_args[0][0] mock_file = mock_open() self.assertTrue(mock_file.write.called) script_contents = mock_file.write.call_args[0][0] self.assertRegex(filename, "job_import-[0-9-]+\.sub$") _, _, filename = filename.rpartition('/') filename, _, _ = filename.rpartition('.') self.assertNotEqual('', filename) # Check that the name in the job file is exactly the same as the script name self.assertIn('#PBS -N {0}'.format(filename), script_contents)
def test_queue_run_system_passes_experiment_to_task(self): mock_open = mock.mock_open() system_id = oid.ObjectId() image_source_id = oid.ObjectId() experiment_id = oid.ObjectId() subject = hpc.HPCJobSystem({}) with mock.patch('batch_analysis.job_systems.hpc_job_system.open', mock_open, create=True): subject.queue_run_system(system_id, image_source_id, experiment_id) mock_file = mock_open() self.assertTrue(mock_file.write.called) script_contents = mock_file.write.call_args[0][0] self.assertIn( "python {0} {1} {2} {3}".format( hpc.quote(task_run_system.__file__), str(system_id), str(image_source_id), str(experiment_id)), script_contents)
def test_queue_benchmark_result_writes_job_script(self): mock_open = mock.mock_open() trial_id = oid.ObjectId() benchmark_id = oid.ObjectId() subject = hpc.HPCJobSystem({}) with mock.patch('batch_analysis.job_systems.hpc_job_system.open', mock_open, create=True): subject.queue_benchmark_result(trial_id, benchmark_id) self.assertTrue(mock_open.called) mock_file = mock_open() self.assertTrue(mock_file.write.called) script_contents = mock_file.write.call_args[0][0] self.assertTrue(script_contents.startswith('#!/bin/bash'), "Did not create a bash script") self.assertIn( "python {0} {1} {2}".format( hpc.quote(task_benchmark_result.__file__), str(trial_id), str(benchmark_id)), script_contents)
def test_queue_benchmark_result_job_name_matches_filename(self): mock_open = mock.mock_open() trial_id = oid.ObjectId() benchmark_id = oid.ObjectId() subject = hpc.HPCJobSystem({}) with mock.patch('batch_analysis.job_systems.hpc_job_system.open', mock_open, create=True): subject.queue_benchmark_result(trial_id, benchmark_id) self.assertTrue(mock_open.called) filename = mock_open.call_args[0][0] mock_file = mock_open() self.assertTrue(mock_file.write.called) script_contents = mock_file.write.call_args[0][0] self.assertRegex(filename, "benchmark-[0-9-]+\.sub$") _, _, filename = filename.rpartition('/') filename, _, _ = filename.rpartition('.') self.assertNotEqual('', filename) # Check that the name in the job file is exactly the same as the script name self.assertIn('#PBS -N {0}'.format(filename), script_contents)
def test_works_with_empty_config(self): hpc.HPCJobSystem({})
def test_queue_generate_dataset_returns_false(self): subject = hpc.HPCJobSystem({}) self.assertFalse(subject.queue_generate_dataset(oid.ObjectId(), {}))
def test_cannot_generate_dataset(self): subject = hpc.HPCJobSystem({}) self.assertFalse(subject.can_generate_dataset(oid.ObjectId(), {}))