def test_job_status(self): # Given/When j = jobs.Job( [sys.executable, '-c', 'import time; time.sleep(0.25)'], output_dir=self.root ) # Then self.assertEqual(j.status(), 'not started') # When j.run() # Then self.assertEqual(j.status(), 'running') # When j.join() self.assertEqual(j.status(), 'done') # Given j = jobs.Job( [sys.executable, '-c', 'asdf'], output_dir=self.root ) # When j.run() j.join() # Then self.assertEqual(j.status(), 'error') self.assertTrue('NameError' in j.get_stderr())
def test_status_when_job_is_rerun(self): # Given command = ['python', '-c', 'print(123)'] j = jobs.Job(command=command, output_dir=self.root) j.run() j.join() # When j1 = jobs.Job(command=command, output_dir=self.root) # Then self.assertEqual(j1.status(), 'done')
def test_simple_job(self): # Given command = [ 'python', '-c', 'import sys;sys.stdout.write("1");sys.stderr.write("2")' ] j = jobs.Job(command=command, output_dir=self.root) # When j.run() j.join() # Then self.assertEqual(j.status(), 'done') self.assertEqual(j.output_dir, self.root) self.assertEqual(j.n_core, 1) self.assertEqual(j.n_thread, 1) self.assertEqual(j.get_stdout(), '1') self.assertEqual(j.get_stderr(), '2') state = j.to_dict() expect = dict(command=command, output_dir=self.root, n_core=1, n_thread=1, env=None) expect['command'][0] = sys.executable self.assertDictEqual(state, expect)
def test_that_job_sets_omp_var(self, mock_total_cores): j = jobs.Job( [ sys.executable, '-c', 'import os;print(os.environ.get("OMP_NUM_THREADS"))' ], output_dir=self.root, n_thread=4, ) j.run() j.join() # Then self.assertEqual(j.status(), 'done') self.assertEqual(j.get_stdout().strip(), '4') self.assertEqual(j.env.get('OMP_NUM_THREADS'), '4') # When j = jobs.Job( [sys.executable, '-c', 'print(1)'], output_dir=self.root, n_thread=None, ) # Then self.assertFalse('OMP_NUM_THREADS' in j.env) # When j = jobs.Job([sys.executable, '-c', 'print(1)'], output_dir=self.root, n_thread=-2, n_core=1) # Then self.assertEqual(j.env.get('OMP_NUM_THREADS'), '2') # When j = jobs.Job([sys.executable, '-c', 'print(1)'], output_dir=self.root, n_thread=-2, n_core=-1) # Then self.assertEqual(j.env.get('OMP_NUM_THREADS'), '4')
def _make_dummy_job(self, n_core=1, sleep=0.05): output = os.path.join(self.root, 'job%d' % self.count) job = jobs.Job( [sys.executable, '-c', 'import time; time.sleep(%f); print(1)' % sleep], output_dir=output, n_core=n_core ) self.count += 1 return job
def test_job_substitute_in_command(self): # Given j = jobs.Job(command=['python', '-c', 'print(123)'], output_dir=self.root) # When sub = '/usr/bin/python' j.substitute_in_command('python', sub) # Then self.assertEqual(j.command[0], sub)
def test_that_job_sets_omp_var(self): j = jobs.Job( [sys.executable, '-c', 'import os;print(os.environ.get("OMP_NUM_THREADS"))'], output_dir=self.root, n_thread=4, ) j.run() j.join() # Then self.assertEqual(j.status(), 'done') self.assertEqual(j.get_stdout().strip(), '4')
def test_clean_removes_new_output_directory(self): # Given out_dir = os.path.join(self.root, 'junk') command = ['python', '-c', 'print(123)'] j = jobs.Job(command=command, output_dir=out_dir) j.run() j.join() # When j.clean() # Then self.assertFalse(os.path.exists(out_dir))
def test_that_job_sets_env_var(self): # Given/When j = jobs.Job( [sys.executable, '-c', 'import os;print(os.environ.get("FOO"))'], output_dir=self.root, env=dict(FOO='hello')) j.run() j.join() # Then print(j.get_stdout(), j.get_stderr()) self.assertEqual(j.status(), 'done') self.assertEqual(j.get_stdout().strip(), 'hello')
def test_status_when_job_is_incorrect(self): j = jobs.Job( [sys.executable, '--junk'], output_dir=self.root, n_thread=4, ) j.run() j.join() # Then self.assertEqual(j.status(), 'error') info = j.get_info() self.assertEqual(info['status'], 'error') self.assertTrue(info['exitcode'] != 0) # Now retry and the status should be the same. j1 = jobs.Job( [sys.executable, '--junk'], output_dir=self.root, n_thread=4, ) self.assertEqual(j1.status(), 'error')
def test_clean_does_not_remove_existing_output_directory(self): # Given command = ['python', '-c', 'print(123)'] j = jobs.Job(command=command, output_dir=self.root) j.run() j.join() # When j.clean() # Then self.assertFalse(os.path.exists(j.stdout)) self.assertFalse(os.path.exists(j.stderr)) self.assertTrue(os.path.exists(self.root))
def test_job_can_handle_string_command(self): # Given command = '''\ python -c 'import sys;sys.stdout.write("1");sys.stderr.write("2")' ''' j = jobs.Job(command=command, output_dir=self.root) # When j.run() j.join() # Then. self.assertTrue(isinstance(j.command, list)) self.assertTrue(j.status(), 'done') self.assertEqual(j.get_stdout(), '1') self.assertEqual(j.get_stderr(), '2')
def test_scheduler_starts_worker_on_submit(self, mock_lw): attrs = {'host': 'localhost', 'free_cores.return_value': 2} mock_lw.return_value = mock.MagicMock(**attrs) # Given config = [dict(host='localhost')] s = jobs.Scheduler(worker_config=config) j = jobs.Job([sys.executable, '-c', 'print(1)'], output_dir=self.root) # When s.submit(j) # Then self.assertEqual(mock_lw.call_count, 1) self.assertEqual(len(s.workers), 1)
def test_scheduler_works_with_local_worker(self, mock_free_cores): # Given s = jobs.Scheduler(worker_config=[dict(host='localhost')]) # When j = jobs.Job( [sys.executable, '-c', 'import time; time.sleep(0.05); print(1)'], output_dir=self.root) proxy = s.submit(j) # Then wait_until(lambda: proxy.status() != 'done', timeout=2) self.assertEqual(proxy.status(), 'done') self.assertEqual(proxy.get_stderr(), '') self.assertEqual(proxy.get_stdout().strip(), '1') info = proxy.get_info() self.assertEqual(info['status'], 'done') self.assertEqual(info['exitcode'], 0)
def test_proc_reset_when_job_done(self): # Given/When j = jobs.Job([sys.executable, '-c', 'import time; time.sleep(0.25)'], output_dir=self.root) # Then self.assertEqual(j.status(), 'not started') self.assertIsNone(j.proc) # When j.run() # Then self.assertEqual(j.status(), 'running') self.assertIsNotNone(j.proc) # When j.join() self.assertEqual(j.status(), 'done') self.assertIsNone(j.proc)
def test_simple(self): # Given r = jobs.RemoteWorker(host='localhost', python=sys.executable, testing=True) # When j = jobs.Job( [sys.executable, '-c', 'import time; time.sleep(0.05); print(1)'], output_dir=self.root) proxy = r.run(j) # Then wait_until(lambda: proxy.status() != 'done') self.assertEqual(proxy.status(), 'done') self.assertEqual(proxy.get_stderr(), '') self.assertEqual(proxy.get_stdout().strip(), '1') info = proxy.get_info() self.assertEqual(info['status'], 'done') self.assertEqual(info['exitcode'], 0)
def test_remote_worker_does_not_copy_when_nfs_is_set(self): # Given r = jobs.RemoteWorker( host='localhost', python=sys.executable, testing=True, nfs=True ) # When j = jobs.Job( [sys.executable, '-c', 'import time; time.sleep(0.05); print(1)'], output_dir=self.root ) proxy = r.run(j) wait_until(lambda: proxy.status() != 'done') # Now set testing to False and check the copying r.testing = False ret = r.copy_output(proxy.job_id, '.') # Then self.assertEqual(ret, None)
def test_reset_proc_when_job_status_error(self): j = jobs.Job( [sys.executable, '--junk'], output_dir=self.root, n_thread=4, ) # Then self.assertEqual(j.status(), 'not started') self.assertIsNone(j.proc) # When j.run() self.assertEqual(j.status(), 'running') self.assertIsNotNone(j.proc) # When j.join() # Then self.assertEqual(j.status(), 'error') self.assertIsNone(j.proc)