def test_make_checkpoint_string(self): '''Test make_checkpoint_string''' bsub = lsf.Job('out', 'error', 'name', 'queue', 1, 'cmd') self.assertEqual('', bsub._make_checkpoint_string()) bsub = lsf.Job('out', 'error', 'name', 'queue', 1, 'cmd', checkpoint=True) self.assertEqual( '-k "' + os.path.abspath('out.checkpoint') + ' method=blcr 600"', bsub._make_checkpoint_string()) bsub = lsf.Job('out', 'error', 'name', 'queue', 1, 'cmd', checkpoint=True, checkpoint_dir='foo', checkpoint_period=42) self.assertEqual('-k "' + os.path.abspath('foo') + ' method=blcr 42"', bsub._make_checkpoint_string())
def test_make_command_string(self): '''Check that command to be bsubbed is made correctly - including for job array''' bsub = lsf.Job('out', 'error', 'name', 'queue', 1, 'cmd') self.assertEqual('cmd', bsub._make_command_string()) bsub = lsf.Job('out', 'error', 'name', 'queue', 1, 'cmd', checkpoint=True) self.assertEqual('cr_run cmd', bsub._make_command_string()) bsub = lsf.Job('out', 'error', 'name', 'queue', 1, 'cmd.INDEX') self.assertEqual('cmd.INDEX', bsub._make_command_string()) bsub = lsf.Job('out', 'error', 'name', 'queue', 1, 'cmd.INDEX foo bar.INDEX', array_start=1, array_end=42) self.assertEqual('cmd.\$LSB_JOBINDEX foo bar.\$LSB_JOBINDEX', bsub._make_command_string()) bsub = lsf.Job('out', 'error', 'name', 'queue', 1, '') with self.assertRaises(lsf.NoCommandGiven): bsub._make_command_string()
def test_make_job_name_string(self): '''Check that the name of the job is set correctly''' bsub = lsf.Job('out', 'error', 'name', 'queue', 1, 'cmd') self.assertEqual('-J name', bsub._make_job_name_string()) bsub = lsf.Job('out', 'error', 'name', 'queue', 1, 'cmd', array_start=1, array_end=42) self.assertEqual('-J "name[1-42]%100"', bsub._make_job_name_string())
def test_run_not_bsubbed(self): '''Test running not bsubbed on normal command''' tmp_out = 'tmp.test_run_not_bsubbed' bsub = lsf.Job('tmp.out', 'tmp.err', 'test', 'normal', 1, 'echo "test_run_not_bsubbed" > ' + tmp_out) bsub.run_not_bsubbed() self.assertTrue( filecmp.cmp( tmp_out, os.path.join(test_dir, 'lsf_unittest_run_not_bsubbed.out'))) os.unlink(tmp_out) bsub = lsf.Job('out', 'error', 'name', 'queue', 1, 'not_a_command_and_should_fail') with self.assertRaises(lsf.Error): bsub.run_not_bsubbed()
def test_make_dependencies_string(self): '''Check that -w 'done()' and 'ended()' are constructed correctly''' bsub = lsf.Job('out', 'error', 'name', 'queue', 1, 'cmd') self.assertEqual('', bsub._make_dependencies_string()) bsub = lsf.Job('out', 'error', 'name', 'queue', 1, 'cmd', depend=42) self.assertEqual("-w 'done(42)'", bsub._make_dependencies_string()) bsub = lsf.Job('out', 'error', 'name', 'queue', 1, 'cmd', depend=[42, 'a']) self.assertEqual("-w 'done(42) && done(\"a\")'", bsub._make_dependencies_string()) bsub = lsf.Job('out', 'error', 'name', 'queue', 1, 'cmd', ended=42) self.assertEqual("-w 'ended(42)'", bsub._make_dependencies_string())
def test_make_resources_string(self): '''Check resources in bsub call get set correctly''' bsub = lsf.Job('out', 'error', 'name', 'queue', 1.5, 'cmd', no_resources=True) self.assertEqual('', bsub._make_resources_string()) # need to manually set the lsf units for these tests bsub = lsf.Job('out', 'error', 'name', 'queue', 1.5, 'cmd') bsub.memory_units = 'MB' resources = bsub._make_resources_string() self.assertEqual('-R "select[mem>1500] rusage[mem=1500]" -M1500', resources) bsub = lsf.Job('out', 'error', 'name', 'queue', 1.5, 'cmd', tokens_name='mytokens', tokens_number=42) bsub.memory_units = 'MB' resources = bsub._make_resources_string() self.assertEqual( '-R "select[mem>1500] rusage[mem=1500,mytokens=42]" -M1500', resources) bsub = lsf.Job('out', 'error', 'name', 'queue', 1.5, 'cmd', threads=42) bsub.memory_units = 'KB' resources = bsub._make_resources_string() self.assertEqual( '-n 42 -R "span[hosts=1] select[mem>1500] rusage[mem=1500]" -M1500000', resources) bsub = lsf.Job('out', 'error', 'name', 'queue', 1, 'cmd', tmp_space=42) bsub.memory_units = 'MB' resources = bsub._make_resources_string() self.assertEqual( '-R "select[mem>1000 && tmp>42000] rusage[mem=1000,tmp=42000]" -M1000', resources)
def test_run(self): '''Test run() runs/fails as expected''' bsub = lsf.Job('out', 'error', 'name', 'queue', 1, 'cmd') bsub._run_test_cmd = os.path.join(test_dir, 'lsf_unittest_run_bsub_fails.sh') with self.assertRaises(lsf.Error): bsub.run() bsub = lsf.Job('out', 'error', 'name', 'queue', 1, 'cmd') bsub._run_test_cmd = 'not_a_command_and_should_fail' with self.assertRaises(lsf.Error): bsub.run() bsub = lsf.Job('out', 'error', 'name', 'queue', 1, 'cmd') bsub._run_test_cmd = os.path.join(test_dir, 'lsf_unittest_run_bsub_ok.sh') bsub.run() self.assertEqual('42', bsub.job_id)
def _make_combine_job(self): self.combine_job = lsf.Job(self.combine_script + '.o', self.combine_script + '.e', self.bsub_name_prefix + '.combine', self.bsub_queue, 0.5, 'bash ' + self.combine_script, memory_units=self.memory_units, threads=2)
def _make_setup_job(self): self.setup_job = lsf.Job( self.setup_script + '.o', self.setup_script + '.e', self.bsub_name_prefix + '.setup', self.bsub_queue, 1, 'bash ' + self.setup_script, memory_units=self.memory_units, )
def _make_array_job(self): self.array_job = lsf.Job('tmp.array.o', 'tmp.array.e', self.bsub_name_prefix + '.array', self.bsub_queue, self.array_mem, self.blast.get_run_command(), array_start=1, array_end=r'''$n''', memory_units=self.memory_units, max_array_size=100)
def test_make_output_files_string(self): '''Check that the names of the stdout and stderr files set properly''' bsub = lsf.Job('out', 'error', 'name', 'queue', 1, 'cmd') self.assertEqual('-o out -e error', bsub._make_output_files_string()) bsub = lsf.Job('out', 'error', 'name', 'queue', 1, 'cmd', array_start=1, array_end=42) self.assertEqual('-o out.%I -e error.%I', bsub._make_output_files_string()) bsub = lsf.Job('doesntexist/out', 'error', 'name', 'queue', 1, 'cmd') with self.assertRaises(lsf.DirectoryDoesNotExist): bsub._make_output_files_string() bsub = lsf.Job('out', 'doesntexist/error', 'name', 'queue', 1, 'cmd') with self.assertRaises(lsf.DirectoryDoesNotExist): bsub._make_output_files_string()
def _make_start_array_job(self): self.start_array_job = lsf.Job( self.start_array_script + '.o', self.start_array_script + '.e', self.bsub_name_prefix + '.start_array', 'small', 0.1, 'bash ' + self.start_array_script, no_resources= True # we have to do this otherwise bmod fails! LSF bug? )
def test_add_dependency(self): bsub = lsf.Job('out', 'error', 'name', 'queue', 1, 'cmd') bsub.add_dependency('jobname') dependencies = ['"jobname"'] self.assertListEqual(bsub.run_when_done, dependencies) bsub.add_dependency(42) dependencies.append('42') self.assertListEqual(bsub.run_when_done, dependencies) bsub.add_dependency('43') dependencies.append('43') self.assertListEqual(bsub.run_when_done, dependencies) bsub.add_dependency(['jobname2', 44]) dependencies.extend(['"jobname2"', '44']) self.assertListEqual(bsub.run_when_done, dependencies) self.assertListEqual(bsub.run_when_ended, []) bsub = lsf.Job('out', 'error', 'name', 'queue', 1, 'cmd') bsub.add_dependency('jobname', ended=True) dependencies = ['"jobname"'] self.assertListEqual(bsub.run_when_ended, dependencies) bsub.add_dependency(42, ended=True) dependencies.append('42') self.assertListEqual(bsub.run_when_ended, dependencies) bsub.add_dependency('43', ended=True) dependencies.append('43') self.assertListEqual(bsub.run_when_ended, dependencies) bsub.add_dependency(['jobname2', 44], ended=True) dependencies.extend(['"jobname2"', '44']) self.assertListEqual(bsub.run_when_ended, dependencies) self.assertListEqual(bsub.run_when_done, [])
def test_set_memory_units(self): '''Check that we can get memory units or die gracefully''' bsub = lsf.Job('out', 'error', 'name', 'queue', 1, 'cmd') with self.assertRaises(lsf.Error): bsub._lsadmin_cmd = 'this_is_not_a_command_and_should_cause_error' bsub._set_memory_units() with self.assertRaises(lsf.Error): bsub._lsadmin_cmd = 'cat ' + os.path.join( test_dir, 'lsf_unittest_lsadmin_showconf_bad_units.txt') bsub._set_memory_units() bsub = lsf.Job('out', 'error', 'name', 'queue', 1, 'cmd') bsub._lsadmin_cmd = 'cat ' + os.path.join( test_dir, 'lsf_unittest_lsadmin_showconf_no_units.txt') bsub._set_memory_units() self.assertEqual('KB', bsub.memory_units) os.environ['FARMPY_LSF_MEMORY_UNITS'] = 'MB' bsub = lsf.Job('out', 'error', 'name', 'queue', 1, 'cmd') bsub._lsadmin_cmd = 'cat ' + os.path.join( test_dir, 'lsf_unittest_lsadmin_showconf_kb.txt') bsub._set_memory_units() self.assertEqual('MB', bsub.memory_units) del os.environ['FARMPY_LSF_MEMORY_UNITS'] bsub = lsf.Job('out', 'error', 'name', 'queue', 1, 'cmd') bsub._lsadmin_cmd = 'cat ' + os.path.join( test_dir, 'lsf_unittest_lsadmin_showconf_kb.txt') bsub._set_memory_units() self.assertEqual('KB', bsub.memory_units) bsub = lsf.Job('out', 'error', 'name', 'queue', 1, 'cmd') bsub._lsadmin_cmd = 'cat ' + os.path.join( test_dir, 'lsf_unittest_lsadmin_showconf_mb.txt') bsub._set_memory_units() self.assertEqual('MB', bsub.memory_units)
def test_set_job_id_from_bsub_output(self): '''Check that stdout from calling bsub is parsed correctly''' bsub = lsf.Job('out', 'error', 'name', 'queue', 1, 'cmd') bsub_out = 'This is not good bsub output\nso should throw error\n' with self.assertRaises(lsf.Error): bsub._set_job_id_from_bsub_output(bsub_out) bsub = lsf.Job('out', 'error', 'name', 'queue', 1, 'cmd') with self.assertRaises(lsf.Error): bsub._set_job_id_from_bsub_output('') bsub = lsf.Job('out', 'error', 'name', 'queue', 1, 'cmd') with self.assertRaises(lsf.Error): bsub._set_job_id_from_bsub_output(None) bsub = lsf.Job('out', 'error', 'name', 'queue', 1, 'cmd') bsub_out = 'Job <42> is submitted to queue <normal>.\nAnother line of output\n' bsub._set_job_id_from_bsub_output(bsub_out) self.assertEqual('42', bsub.job_id) bsub = lsf.Job('out', 'error', 'name', 'queue', 1, 'cmd') bsub_out = 'Job <not_an_int> is submitted to queue <normal>\n' with self.assertRaises(lsf.Error): bsub._set_job_id_from_bsub_output(bsub_out)
def test_run_not_bsubbed_array(self): '''Test running not bsubbed on job array''' tmp_out = 'tmp.test_run_not_bsubbed_array' bsub = lsf.Job('tmp.out', 'tmp.err', 'test', 'normal', 1, 'echo "test_run_not_bsubbed array INDEX" >> ' + tmp_out, array_start=1, array_end=3) bsub.run_not_bsubbed() self.assertTrue( filecmp.cmp( tmp_out, os.path.join(test_dir, 'lsf_unittest_run_not_bsubbed_array.out'))) os.unlink(tmp_out)
options.err = options.name + '.e' if options.out is None: options.out = options.name + '.o' # make bsub object and run it b = lsf.Job(options.out, options.err, options.name, options.queue, options.memory, command, checkpoint=options.checkpoint, checkpoint_dir=options.checkpoint_dir, checkpoint_period=options.checkpoint_period, array_start=options.start, array_end=options.end, depend=options.done, threads=options.threads, tmp_space=options.tmp_space, ended=options.ended, tokens_name=options.tokens_name, tokens_number=options.tokens_number, memory_units=options.memory_units, max_array_size=options.array_limit) print(b) if not options.norun: b.run() print(b.job_id, 'submitted')
def test_make_queue_string(self): '''Check that queue set correctly''' bsub = lsf.Job('out', 'error', 'name', 'queue', 1, 'cmd') self.assertEqual('-q queue', bsub._make_queue_string()) bsub = lsf.Job('out', 'error', 'name', None, 1, 'cmd') self.assertEqual('', bsub._make_queue_string())
def test_make_prexec_test_string(self): bsub = lsf.Job('out', 'error', 'name', 'queue', 1, 'cmd') home = os.path.expanduser('~') self.assertEqual("-E 'test -e " + home + "'", bsub._make_prexec_test_string())