def test_job_with_job_dep(self, patcher): patcher.side_effect = ['Your job 3891651 ("MV1") has been submitted'] job = _job.Job( SUBCOMMAND.VALIDATE, queue='all', output_dir='output_dir', script='script.sh', name='MV1', memory_limit=1, mail_user='******', mail_type=_constants.MAIL_TYPE.ALL, ) dep = _job.Job(job_ident='1234', output_dir='', stage=SUBCOMMAND.VALIDATE) job.dependencies.append(dep) sched = _scheduler.SgeScheduler() sched.submit(job) self.assertEqual('3891651', job.job_ident) self.assertEqual(_constants.JOB_STATUS.SUBMITTED, job.status) patcher.assert_called_with( 'qsub -j y -q all -l mem_free=1M,mem_token=1M,h_vmem=1M -l h_rt=16:00:00 -V ' '-hold_jid 1234 -N MV1 -m abes -M [email protected] ' '-o output_dir/job-\\$JOB_NAME-\\$JOB_ID.log script.sh', shell=True, )
def test_dependent_job(self, patch_check): patch_check.side_effect = [ "Submitted batch job 1665695".encode('utf8') ] job = _job.Job(output_dir='temp', name='job1', stage='validate', script='submit.sh', dependencies=[ _job.Job(output_dir='temp', name='job2', stage='cluster', script='submit2.sh', job_ident='12345678') ]) print(job) _scheduler.SlurmScheduler().submit(job) self.assertEqual(_constants.JOB_STATUS.SUBMITTED, job.status) self.assertEqual('1665695', job.job_ident) patch_check.assert_called_with([ 'sbatch', '--mem', '16000M', '-t', '16:00:00', '--export=ALL', '--dependency=afterok:12345678', '-J', 'job1', '-o', 'temp/job-%x-%j.log', 'submit.sh' ], shell=False)
def test_job_with_mixed_deps(self, patcher): patcher.side_effect = ['141.torque01.bcgsc.ca\n'] job = _job.Job(stage=SUBCOMMAND.VALIDATE, queue='all', output_dir='output_dir', name='MV1', memory_limit=1, mail_user='******', mail_type=_constants.MAIL_TYPE.ALL, script='script.sh', dependencies=[ _job.Job(stage=SUBCOMMAND.VALIDATE, output_dir='output_dir', job_ident='1234.torque01.bcgsc.ca'), _job.Job(stage=SUBCOMMAND.VALIDATE, output_dir='output_dir', job_ident='54.torque01.bcgsc.ca'), _job.TorqueArrayJob( stage=SUBCOMMAND.VALIDATE, output_dir='output_dir', job_ident='99[].torque01.bcgsc.ca', task_list=5) ]) sched = _scheduler.TorqueScheduler() sched.submit(job) self.assertEqual('141.torque01.bcgsc.ca', job.job_ident) patcher.assert_called_with([ 'qsub', '-j', 'oe', '-q', 'all', '-l', 'mem=1mb', '-l', 'walltime=16:00:00', '-V', '-W depend=afterokarray:99[][5].torque01.bcgsc.ca,afterok:1234.torque01.bcgsc.ca:54.torque01.bcgsc.ca', '-N', 'MV1', '-o', 'output_dir/job-$PBS_JOBNAME-$PBS_JOBID.log', '-m', 'abef', '-M', '*****@*****.**', 'script.sh' ])
def test_bad_command(self, patcher): patcher.side_effect = [subprocess.CalledProcessError(1, 'cmd')] sched = _scheduler.SlurmScheduler() job = _job.Job(SUBCOMMAND.VALIDATE, '', job_ident='1234') with self.assertRaises(subprocess.CalledProcessError): sched.cancel(job) patcher.assert_called_with(['scancel', '1234'])
def test_mail_options(self, patch_check): patch_check.return_value = "Submitted batch job 1665695".encode('utf8') job = _job.Job( output_dir='temp', name='job1', stage='validate', script='submit.sh', mail_user='******', mail_type=_constants.MAIL_TYPE.ALL, ) print(job) _scheduler.SlurmScheduler().submit(job) self.assertEqual(_constants.JOB_STATUS.SUBMITTED, job.status) self.assertEqual('1665695', job.job_ident) patch_check.assert_called_with( [ 'sbatch', '--mem', '16000M', '-t', '16:00:00', '--export=ALL', '-J', 'job1', '-o', 'temp/job-%x-%j.log', '--mail-type=ALL', '[email protected]', 'submit.sh', ], shell=False, )
def test_partition(self, patch_check): patch_check.return_value = "Submitted batch job 1665695".encode('utf8') job = _job.Job( output_dir='temp', name='job1', stage='validate', script='submit.sh', queue='all' ) print(job) _scheduler.SlurmScheduler().submit(job) self.assertEqual(_constants.JOB_STATUS.SUBMITTED, job.status) self.assertEqual('1665695', job.job_ident) patch_check.assert_called_with( [ 'sbatch', '--partition=all', '--mem', '16000M', '-t', '16:00:00', '--export=ALL', '-J', 'job1', '-o', 'temp/job-%x-%j.log', 'submit.sh', ], shell=False, )
def test_bad_command(self, patcher): patcher.side_effect = [subprocess.CalledProcessError(1, 'cmd')] sched = _scheduler.TorqueScheduler() job = _job.Job(SUBCOMMAND.VALIDATE, '', job_ident='1234') sched.cancel(job) patcher.assert_called_with(['qdel', '1234']) self.assertNotEqual(_constants.JOB_STATUS.CANCELLED, job.status)
def test_already_submitted_error(self): job = _job.Job(stage=SUBCOMMAND.VALIDATE, output_dir='output_dir', job_ident='1') sched = _scheduler.SgeScheduler(concurrency_limit=2) with self.assertRaises(ValueError): sched.submit(job)
def test_array_job_with_job_dep(self, patcher): patcher.side_effect = [ 'Your job-array 3891657.2-4:1 ("MV1") has been submitted' ] job = _job.ArrayJob( stage=SUBCOMMAND.VALIDATE, output_dir='output_dir', script='script.sh', name='MV1', task_list=[2, 3, 4], memory_limit=1, ) sched = _scheduler.SgeScheduler(concurrency_limit=2) dep = _job.Job(job_ident='1234', output_dir='', stage=SUBCOMMAND.VALIDATE) job.dependencies.append(dep) sched.submit(job) self.assertEqual('3891657', job.job_ident) self.assertEqual(_constants.JOB_STATUS.SUBMITTED, job.status) patcher.assert_called_with( 'qsub -j y -l mem_free=1M,mem_token=1M,h_vmem=1M -l h_rt=16:00:00 -V ' '-hold_jid 1234 ' '-N MV1 -t 2-4 -o output_dir/job-\\$JOB_NAME-\\$JOB_ID-\\$TASK_ID.log script.sh', shell=True, )
def test_dependency_error(self, patch_check): patch_check.side_effect = [ "Submitted batch job 12345678".encode('utf8'), "Submitted batch job 1665695".encode('utf8'), ] job = _job.Job( output_dir='temp', name='job1', stage='validate', script='submit.sh', dependencies=[ _job.Job(output_dir='temp', name='job2', stage='cluster', script='submit2.sh') ], ) print(job) with self.assertRaises(ValueError): _scheduler.SlurmScheduler().submit(job)
def test_single_job(self, patch_check): content = """ job-ID prior name user state submit/start at queue slots ja-task-ID ----------------------------------------------------------------------------------------------------------------- 217940 1.50000 subtest.sh creisle qw 05/22/2018 23:39:55 1 """.encode('utf8') patch_check.return_value = content job = _job.Job(output_dir='temp', job_ident='217940', stage='validate') _scheduler.SgeScheduler().update_info(job) self.assertEqual(_constants.JOB_STATUS.PENDING, job.status)
def test_job(self, patcher): patcher.side_effect = ['Your job 3891651 ("MV1") has been submitted'] job = _job.Job( SUBCOMMAND.VALIDATE, queue='all', output_dir='output_dir', script='script.sh', name='MV1', memory_limit=1 ) sched = _scheduler.SgeScheduler() sched.submit(job) self.assertEqual('3891651', job.job_ident) self.assertEqual(_constants.JOB_STATUS.SUBMITTED, job.status) patcher.assert_called_with( 'qsub -j y -q all -l mem_free=1M,mem_token=1M,h_vmem=1M -l h_rt=16:00:00 -V ' '-N MV1 -o output_dir/job-\\$JOB_NAME-\\$JOB_ID.log script.sh', shell=True)
def test_job(self, patcher): patcher.side_effect = ['141.torque01.bcgsc.ca\n'] job = _job.Job(stage=SUBCOMMAND.VALIDATE, queue='all', output_dir='output_dir', name='MV1', memory_limit=1, mail_user='******', mail_type=_constants.MAIL_TYPE.ALL, script='script.sh') sched = _scheduler.TorqueScheduler() sched.submit(job) self.assertEqual('141.torque01.bcgsc.ca', job.job_ident) patcher.assert_called_with([ 'qsub', '-j', 'oe', '-q', 'all', '-l', 'mem=1mb', '-l', 'walltime=16:00:00', '-V', '-N', 'MV1', '-o', 'output_dir/job-$PBS_JOBNAME-$PBS_JOBID.log', '-m', 'abef', '-M', '*****@*****.**', 'script.sh' ])
def test_single_job(self, patcher): sched = _scheduler.SlurmScheduler() job = _job.Job(SUBCOMMAND.VALIDATE, '', job_ident='1234') sched.cancel(job) self.assertEqual(_constants.JOB_STATUS.CANCELLED, job.status) patcher.assert_called_with(['scancel', '1234'])
def test_bad_command(self, patcher): patcher.side_effect = [subprocess.CalledProcessError(1, 'command')] sched = _scheduler.SgeScheduler() job = _job.Job(SUBCOMMAND.VALIDATE, '', job_ident='1234') sched.cancel(job) self.assertEqual(_constants.JOB_STATUS.NOT_SUBMITTED, job.status)