Example #1
0
    def test_job_status(self):
        # Given/When
        j = jobs.Job(
            [sys.executable, '-c', 'import time; time.sleep(0.25)'],
            output_dir=self.root
        )

        # Then
        self.assertEqual(j.status(), 'not started')

        # When
        j.run()

        # Then
        self.assertEqual(j.status(), 'running')

        # When
        j.join()
        self.assertEqual(j.status(), 'done')

        # Given
        j = jobs.Job(
            [sys.executable, '-c', 'asdf'],
            output_dir=self.root
        )
        # When
        j.run()
        j.join()

        # Then
        self.assertEqual(j.status(), 'error')
        self.assertTrue('NameError' in j.get_stderr())
Example #2
0
    def test_status_when_job_is_rerun(self):
        # Given
        command = ['python', '-c', 'print(123)']
        j = jobs.Job(command=command, output_dir=self.root)
        j.run()
        j.join()

        # When
        j1 = jobs.Job(command=command, output_dir=self.root)

        # Then
        self.assertEqual(j1.status(), 'done')
Example #3
0
    def test_simple_job(self):
        # Given
        command = [
            'python', '-c',
            'import sys;sys.stdout.write("1");sys.stderr.write("2")'
        ]
        j = jobs.Job(command=command, output_dir=self.root)

        # When
        j.run()
        j.join()

        # Then
        self.assertEqual(j.status(), 'done')
        self.assertEqual(j.output_dir, self.root)
        self.assertEqual(j.n_core, 1)
        self.assertEqual(j.n_thread, 1)
        self.assertEqual(j.get_stdout(), '1')
        self.assertEqual(j.get_stderr(), '2')
        state = j.to_dict()
        expect = dict(command=command,
                      output_dir=self.root,
                      n_core=1,
                      n_thread=1,
                      env=None)
        expect['command'][0] = sys.executable
        self.assertDictEqual(state, expect)
Example #4
0
    def test_that_job_sets_omp_var(self, mock_total_cores):
        j = jobs.Job(
            [
                sys.executable, '-c',
                'import os;print(os.environ.get("OMP_NUM_THREADS"))'
            ],
            output_dir=self.root,
            n_thread=4,
        )
        j.run()
        j.join()

        # Then
        self.assertEqual(j.status(), 'done')
        self.assertEqual(j.get_stdout().strip(), '4')
        self.assertEqual(j.env.get('OMP_NUM_THREADS'), '4')

        # When
        j = jobs.Job(
            [sys.executable, '-c', 'print(1)'],
            output_dir=self.root,
            n_thread=None,
        )

        # Then
        self.assertFalse('OMP_NUM_THREADS' in j.env)

        # When
        j = jobs.Job([sys.executable, '-c', 'print(1)'],
                     output_dir=self.root,
                     n_thread=-2,
                     n_core=1)

        # Then
        self.assertEqual(j.env.get('OMP_NUM_THREADS'), '2')

        # When
        j = jobs.Job([sys.executable, '-c', 'print(1)'],
                     output_dir=self.root,
                     n_thread=-2,
                     n_core=-1)

        # Then
        self.assertEqual(j.env.get('OMP_NUM_THREADS'), '4')
Example #5
0
 def _make_dummy_job(self, n_core=1, sleep=0.05):
     output = os.path.join(self.root, 'job%d' % self.count)
     job = jobs.Job(
         [sys.executable, '-c',
          'import time; time.sleep(%f); print(1)' % sleep],
         output_dir=output,
         n_core=n_core
     )
     self.count += 1
     return job
Example #6
0
    def test_job_substitute_in_command(self):
        # Given
        j = jobs.Job(command=['python', '-c',  'print(123)'],
                     output_dir=self.root)

        # When
        sub = '/usr/bin/python'
        j.substitute_in_command('python', sub)

        # Then
        self.assertEqual(j.command[0], sub)
Example #7
0
    def test_that_job_sets_omp_var(self):
        j = jobs.Job(
            [sys.executable, '-c',
             'import os;print(os.environ.get("OMP_NUM_THREADS"))'],
            output_dir=self.root,
            n_thread=4,
        )
        j.run()
        j.join()

        # Then
        self.assertEqual(j.status(), 'done')
        self.assertEqual(j.get_stdout().strip(), '4')
Example #8
0
    def test_clean_removes_new_output_directory(self):
        # Given
        out_dir = os.path.join(self.root, 'junk')
        command = ['python', '-c', 'print(123)']
        j = jobs.Job(command=command, output_dir=out_dir)
        j.run()
        j.join()

        # When
        j.clean()

        # Then
        self.assertFalse(os.path.exists(out_dir))
Example #9
0
    def test_that_job_sets_env_var(self):
        # Given/When
        j = jobs.Job(
            [sys.executable, '-c', 'import os;print(os.environ.get("FOO"))'],
            output_dir=self.root,
            env=dict(FOO='hello'))
        j.run()
        j.join()

        # Then
        print(j.get_stdout(), j.get_stderr())
        self.assertEqual(j.status(), 'done')
        self.assertEqual(j.get_stdout().strip(), 'hello')
Example #10
0
    def test_status_when_job_is_incorrect(self):
        j = jobs.Job(
            [sys.executable, '--junk'],
            output_dir=self.root,
            n_thread=4,
        )
        j.run()
        j.join()

        # Then
        self.assertEqual(j.status(), 'error')
        info = j.get_info()
        self.assertEqual(info['status'], 'error')
        self.assertTrue(info['exitcode'] != 0)

        # Now retry and the status should be the same.
        j1 = jobs.Job(
            [sys.executable, '--junk'],
            output_dir=self.root,
            n_thread=4,
        )

        self.assertEqual(j1.status(), 'error')
Example #11
0
    def test_clean_does_not_remove_existing_output_directory(self):
        # Given
        command = ['python', '-c', 'print(123)']
        j = jobs.Job(command=command, output_dir=self.root)
        j.run()
        j.join()

        # When
        j.clean()

        # Then
        self.assertFalse(os.path.exists(j.stdout))
        self.assertFalse(os.path.exists(j.stderr))
        self.assertTrue(os.path.exists(self.root))
Example #12
0
    def test_job_can_handle_string_command(self):
        # Given
        command = '''\
        python -c 'import sys;sys.stdout.write("1");sys.stderr.write("2")' '''
        j = jobs.Job(command=command, output_dir=self.root)

        # When
        j.run()
        j.join()

        # Then.
        self.assertTrue(isinstance(j.command, list))
        self.assertTrue(j.status(), 'done')
        self.assertEqual(j.get_stdout(), '1')
        self.assertEqual(j.get_stderr(), '2')
Example #13
0
    def test_scheduler_starts_worker_on_submit(self, mock_lw):
        attrs = {'host': 'localhost', 'free_cores.return_value': 2}
        mock_lw.return_value = mock.MagicMock(**attrs)

        # Given
        config = [dict(host='localhost')]
        s = jobs.Scheduler(worker_config=config)
        j = jobs.Job([sys.executable, '-c', 'print(1)'], output_dir=self.root)

        # When
        s.submit(j)

        # Then
        self.assertEqual(mock_lw.call_count, 1)
        self.assertEqual(len(s.workers), 1)
Example #14
0
    def test_scheduler_works_with_local_worker(self, mock_free_cores):
        # Given
        s = jobs.Scheduler(worker_config=[dict(host='localhost')])

        # When
        j = jobs.Job(
            [sys.executable, '-c', 'import time; time.sleep(0.05); print(1)'],
            output_dir=self.root)
        proxy = s.submit(j)

        # Then
        wait_until(lambda: proxy.status() != 'done', timeout=2)
        self.assertEqual(proxy.status(), 'done')
        self.assertEqual(proxy.get_stderr(), '')
        self.assertEqual(proxy.get_stdout().strip(), '1')
        info = proxy.get_info()
        self.assertEqual(info['status'], 'done')
        self.assertEqual(info['exitcode'], 0)
Example #15
0
    def test_proc_reset_when_job_done(self):
        # Given/When
        j = jobs.Job([sys.executable, '-c', 'import time; time.sleep(0.25)'],
                     output_dir=self.root)

        # Then
        self.assertEqual(j.status(), 'not started')
        self.assertIsNone(j.proc)

        # When
        j.run()

        # Then
        self.assertEqual(j.status(), 'running')
        self.assertIsNotNone(j.proc)

        # When
        j.join()
        self.assertEqual(j.status(), 'done')
        self.assertIsNone(j.proc)
Example #16
0
    def test_simple(self):
        # Given
        r = jobs.RemoteWorker(host='localhost',
                              python=sys.executable,
                              testing=True)

        # When
        j = jobs.Job(
            [sys.executable, '-c', 'import time; time.sleep(0.05); print(1)'],
            output_dir=self.root)
        proxy = r.run(j)

        # Then
        wait_until(lambda: proxy.status() != 'done')
        self.assertEqual(proxy.status(), 'done')
        self.assertEqual(proxy.get_stderr(), '')
        self.assertEqual(proxy.get_stdout().strip(), '1')
        info = proxy.get_info()
        self.assertEqual(info['status'], 'done')
        self.assertEqual(info['exitcode'], 0)
Example #17
0
    def test_remote_worker_does_not_copy_when_nfs_is_set(self):
        # Given
        r = jobs.RemoteWorker(
            host='localhost', python=sys.executable, testing=True, nfs=True
        )

        # When
        j = jobs.Job(
            [sys.executable, '-c', 'import time; time.sleep(0.05); print(1)'],
            output_dir=self.root
        )
        proxy = r.run(j)
        wait_until(lambda: proxy.status() != 'done')

        # Now set testing to False and check the copying
        r.testing = False
        ret = r.copy_output(proxy.job_id, '.')

        # Then
        self.assertEqual(ret, None)
Example #18
0
    def test_reset_proc_when_job_status_error(self):
        j = jobs.Job(
            [sys.executable, '--junk'],
            output_dir=self.root,
            n_thread=4,
        )

        # Then
        self.assertEqual(j.status(), 'not started')
        self.assertIsNone(j.proc)

        # When
        j.run()

        self.assertEqual(j.status(), 'running')
        self.assertIsNotNone(j.proc)

        # When
        j.join()

        # Then
        self.assertEqual(j.status(), 'error')
        self.assertIsNone(j.proc)