def test_job_with_job_dep(self, patcher):
patcher.side_effect = ['Your job 3891651 ("MV1") has been submitted']
job = _job.Job(
SUBCOMMAND.VALIDATE,
queue='all',
output_dir='output_dir',
script='script.sh',
name='MV1',
memory_limit=1,
mail_user='******',
mail_type=_constants.MAIL_TYPE.ALL,
)
dep = _job.Job(job_ident='1234',
output_dir='',
stage=SUBCOMMAND.VALIDATE)
job.dependencies.append(dep)
sched = _scheduler.SgeScheduler()
sched.submit(job)
self.assertEqual('3891651', job.job_ident)
self.assertEqual(_constants.JOB_STATUS.SUBMITTED, job.status)
patcher.assert_called_with(
'qsub -j y -q all -l mem_free=1M,mem_token=1M,h_vmem=1M -l h_rt=16:00:00 -V '
'-hold_jid 1234 -N MV1 -m abes -M [email protected] '
'-o output_dir/job-\\$JOB_NAME-\\$JOB_ID.log script.sh',
shell=True,
)
def test_dependent_job(self, patch_check):
patch_check.side_effect = [
"Submitted batch job 1665695".encode('utf8')
]
job = _job.Job(output_dir='temp',
name='job1',
stage='validate',
script='submit.sh',
dependencies=[
_job.Job(output_dir='temp',
name='job2',
stage='cluster',
script='submit2.sh',
job_ident='12345678')
])
print(job)
_scheduler.SlurmScheduler().submit(job)
self.assertEqual(_constants.JOB_STATUS.SUBMITTED, job.status)
self.assertEqual('1665695', job.job_ident)
patch_check.assert_called_with([
'sbatch', '--mem', '16000M', '-t', '16:00:00', '--export=ALL',
'--dependency=afterok:12345678', '-J', 'job1', '-o',
'temp/job-%x-%j.log', 'submit.sh'
],
shell=False)
def test_job_with_mixed_deps(self, patcher):
patcher.side_effect = ['141.torque01.bcgsc.ca\n']
job = _job.Job(stage=SUBCOMMAND.VALIDATE,
queue='all',
output_dir='output_dir',
name='MV1',
memory_limit=1,
mail_user='******',
mail_type=_constants.MAIL_TYPE.ALL,
script='script.sh',
dependencies=[
_job.Job(stage=SUBCOMMAND.VALIDATE,
output_dir='output_dir',
job_ident='1234.torque01.bcgsc.ca'),
_job.Job(stage=SUBCOMMAND.VALIDATE,
output_dir='output_dir',
job_ident='54.torque01.bcgsc.ca'),
_job.TorqueArrayJob(
stage=SUBCOMMAND.VALIDATE,
output_dir='output_dir',
job_ident='99[].torque01.bcgsc.ca',
task_list=5)
])
sched = _scheduler.TorqueScheduler()
sched.submit(job)
self.assertEqual('141.torque01.bcgsc.ca', job.job_ident)
patcher.assert_called_with([
'qsub', '-j', 'oe', '-q', 'all', '-l', 'mem=1mb', '-l',
'walltime=16:00:00', '-V',
'-W depend=afterokarray:99[][5].torque01.bcgsc.ca,afterok:1234.torque01.bcgsc.ca:54.torque01.bcgsc.ca',
'-N', 'MV1', '-o', 'output_dir/job-$PBS_JOBNAME-$PBS_JOBID.log',
'-m', 'abef', '-M', '*****@*****.**', 'script.sh'
])
def test_bad_command(self, patcher):
patcher.side_effect = [subprocess.CalledProcessError(1, 'cmd')]
sched = _scheduler.SlurmScheduler()
job = _job.Job(SUBCOMMAND.VALIDATE, '', job_ident='1234')
with self.assertRaises(subprocess.CalledProcessError):
sched.cancel(job)
patcher.assert_called_with(['scancel', '1234'])
def test_mail_options(self, patch_check):
patch_check.return_value = "Submitted batch job 1665695".encode('utf8')
job = _job.Job(
output_dir='temp',
name='job1',
stage='validate',
script='submit.sh',
mail_user='******',
mail_type=_constants.MAIL_TYPE.ALL,
)
print(job)
_scheduler.SlurmScheduler().submit(job)
self.assertEqual(_constants.JOB_STATUS.SUBMITTED, job.status)
self.assertEqual('1665695', job.job_ident)
patch_check.assert_called_with(
[
'sbatch',
'--mem',
'16000M',
'-t',
'16:00:00',
'--export=ALL',
'-J',
'job1',
'-o',
'temp/job-%x-%j.log',
'--mail-type=ALL',
'[email protected]',
'submit.sh',
],
shell=False,
)
def test_partition(self, patch_check):
patch_check.return_value = "Submitted batch job 1665695".encode('utf8')
job = _job.Job(
output_dir='temp', name='job1', stage='validate', script='submit.sh', queue='all'
)
print(job)
_scheduler.SlurmScheduler().submit(job)
self.assertEqual(_constants.JOB_STATUS.SUBMITTED, job.status)
self.assertEqual('1665695', job.job_ident)
patch_check.assert_called_with(
[
'sbatch',
'--partition=all',
'--mem',
'16000M',
'-t',
'16:00:00',
'--export=ALL',
'-J',
'job1',
'-o',
'temp/job-%x-%j.log',
'submit.sh',
],
shell=False,
)
def test_bad_command(self, patcher):
patcher.side_effect = [subprocess.CalledProcessError(1, 'cmd')]
sched = _scheduler.TorqueScheduler()
job = _job.Job(SUBCOMMAND.VALIDATE, '', job_ident='1234')
sched.cancel(job)
patcher.assert_called_with(['qdel', '1234'])
self.assertNotEqual(_constants.JOB_STATUS.CANCELLED, job.status)
def test_already_submitted_error(self):
job = _job.Job(stage=SUBCOMMAND.VALIDATE,
output_dir='output_dir',
job_ident='1')
sched = _scheduler.SgeScheduler(concurrency_limit=2)
with self.assertRaises(ValueError):
sched.submit(job)
def test_array_job_with_job_dep(self, patcher):
patcher.side_effect = [
'Your job-array 3891657.2-4:1 ("MV1") has been submitted'
]
job = _job.ArrayJob(
stage=SUBCOMMAND.VALIDATE,
output_dir='output_dir',
script='script.sh',
name='MV1',
task_list=[2, 3, 4],
memory_limit=1,
)
sched = _scheduler.SgeScheduler(concurrency_limit=2)
dep = _job.Job(job_ident='1234',
output_dir='',
stage=SUBCOMMAND.VALIDATE)
job.dependencies.append(dep)
sched.submit(job)
self.assertEqual('3891657', job.job_ident)
self.assertEqual(_constants.JOB_STATUS.SUBMITTED, job.status)
patcher.assert_called_with(
'qsub -j y -l mem_free=1M,mem_token=1M,h_vmem=1M -l h_rt=16:00:00 -V '
'-hold_jid 1234 '
'-N MV1 -t 2-4 -o output_dir/job-\\$JOB_NAME-\\$JOB_ID-\\$TASK_ID.log script.sh',
shell=True,
)
def test_dependency_error(self, patch_check):
patch_check.side_effect = [
"Submitted batch job 12345678".encode('utf8'),
"Submitted batch job 1665695".encode('utf8'),
]
job = _job.Job(
output_dir='temp',
name='job1',
stage='validate',
script='submit.sh',
dependencies=[
_job.Job(output_dir='temp', name='job2', stage='cluster', script='submit2.sh')
],
)
print(job)
with self.assertRaises(ValueError):
_scheduler.SlurmScheduler().submit(job)
def test_single_job(self, patch_check):
content = """
job-ID prior name user state submit/start at queue slots ja-task-ID
-----------------------------------------------------------------------------------------------------------------
217940 1.50000 subtest.sh creisle qw 05/22/2018 23:39:55 1
""".encode('utf8')
patch_check.return_value = content
job = _job.Job(output_dir='temp', job_ident='217940', stage='validate')
_scheduler.SgeScheduler().update_info(job)
self.assertEqual(_constants.JOB_STATUS.PENDING, job.status)
def test_job(self, patcher):
patcher.side_effect = ['Your job 3891651 ("MV1") has been submitted']
job = _job.Job(
SUBCOMMAND.VALIDATE,
queue='all',
output_dir='output_dir',
script='script.sh',
name='MV1',
memory_limit=1
)
sched = _scheduler.SgeScheduler()
sched.submit(job)
self.assertEqual('3891651', job.job_ident)
self.assertEqual(_constants.JOB_STATUS.SUBMITTED, job.status)
patcher.assert_called_with(
'qsub -j y -q all -l mem_free=1M,mem_token=1M,h_vmem=1M -l h_rt=16:00:00 -V '
'-N MV1 -o output_dir/job-\\$JOB_NAME-\\$JOB_ID.log script.sh', shell=True)
def test_job(self, patcher):
patcher.side_effect = ['141.torque01.bcgsc.ca\n']
job = _job.Job(stage=SUBCOMMAND.VALIDATE,
queue='all',
output_dir='output_dir',
name='MV1',
memory_limit=1,
mail_user='******',
mail_type=_constants.MAIL_TYPE.ALL,
script='script.sh')
sched = _scheduler.TorqueScheduler()
sched.submit(job)
self.assertEqual('141.torque01.bcgsc.ca', job.job_ident)
patcher.assert_called_with([
'qsub', '-j', 'oe', '-q', 'all', '-l', 'mem=1mb', '-l',
'walltime=16:00:00', '-V', '-N', 'MV1', '-o',
'output_dir/job-$PBS_JOBNAME-$PBS_JOBID.log', '-m', 'abef', '-M',
'*****@*****.**', 'script.sh'
])
def test_single_job(self, patcher):
sched = _scheduler.SlurmScheduler()
job = _job.Job(SUBCOMMAND.VALIDATE, '', job_ident='1234')
sched.cancel(job)
self.assertEqual(_constants.JOB_STATUS.CANCELLED, job.status)
patcher.assert_called_with(['scancel', '1234'])
def test_bad_command(self, patcher):
patcher.side_effect = [subprocess.CalledProcessError(1, 'command')]
sched = _scheduler.SgeScheduler()
job = _job.Job(SUBCOMMAND.VALIDATE, '', job_ident='1234')
sched.cancel(job)
self.assertEqual(_constants.JOB_STATUS.NOT_SUBMITTED, job.status)
