def test_mail_options(self, patch_check): patch_check.return_value = "Submitted batch job 1665695".encode('utf8') job = _job.Job( output_dir='temp', name='job1', stage='validate', script='submit.sh', mail_user='******', mail_type=_constants.MAIL_TYPE.ALL, ) print(job) _scheduler.SlurmScheduler().submit(job) self.assertEqual(_constants.JOB_STATUS.SUBMITTED, job.status) self.assertEqual('1665695', job.job_ident) patch_check.assert_called_with( [ 'sbatch', '--mem', '16000M', '-t', '16:00:00', '--export=ALL', '-J', 'job1', '-o', 'temp/job-%x-%j.log', '--mail-type=ALL', '[email protected]', 'submit.sh', ], shell=False, )
def test_cancelled_task(self): content = """ JobId=1697512 ArrayJobId=1697503 ArrayTaskId=1 JobName=MV_mock-A47933_batch-uwSwW68EW43XNdvq85NxJ7 UserId=creisle(1365) GroupId=users(100) MCS_label=N/A Priority=42 Nice=0 Account=all QOS=normal JobState=CANCELLED Reason=None Dependency=(null) Requeue=1 Restarts=0 BatchFlag=1 Reboot=0 ExitCode=0:15 RunTime=00:00:02 TimeLimit=16:00:00 TimeMin=N/A SubmitTime=2018-05-31T20:01:46 EligibleTime=2018-05-31T20:01:49 StartTime=2018-05-31T20:02:05 EndTime=2018-05-31T20:02:07 Deadline=N/A PreemptTime=None SuspendTime=None SecsPreSuspend=0 Partition=all AllocNode:Sid=n104:173998 ReqNodeList=(null) ExcNodeList=(null) NodeList=n245 BatchHost=n245 NumNodes=1 NumCPUs=1 NumTasks=0 CPUs/Task=1 ReqB:S:C:T=0:0:*:* TRES=cpu=1,mem=18000M,node=1 Socks/Node=* NtasksPerN:B:S:C=0:0:*:* CoreSpec=* MinCPUsNode=1 MinMemoryNode=18000M MinTmpDiskNode=0 Features=(null) DelayBoot=00:00:00 Gres=(null) Reservation=(null) OverSubscribe=OK Contiguous=0 Licenses=(null) Network=(null) Command=/projects/trans_scratch/validations/workspace/creisle/temp/test_submission/slurm/mock-A47933_diseased_transcriptome/validate/submit.sh WorkDir=/home/creisle StdErr=/projects/trans_scratch/validations/workspace/creisle/temp/test_submission/slurm/mock-A47933_diseased_transcriptome/validate/batch-uwSwW68EW43XNdvq85NxJ7-1/job-%x-1697503-1.log StdIn=/dev/null StdOut=/projects/trans_scratch/validations/workspace/creisle/temp/test_submission/slurm/mock-A47933_diseased_transcriptome/validate/batch-uwSwW68EW43XNdvq85NxJ7-1/job-%x-1697503-1.log Power= """ rows = _scheduler.SlurmScheduler().parse_scontrol_show(content) self.assertEqual(1, len(rows)) row = rows[0] self.assertEqual(_constants.JOB_STATUS.CANCELLED, row['status'])
def test_job_array_concurrency_limit(self, patch_check): patch_check.side_effect = ["Submitted batch job 1665695".encode('utf8')] print(patch_check) job = _job.ArrayJob( output_dir='temp', name='job1', stage='validate', script='submit.sh', task_list=[1, 2, 3, 4, 5, 14, 16], ) _scheduler.SlurmScheduler(concurrency_limit=2).submit(job) self.assertEqual(_constants.JOB_STATUS.SUBMITTED, job.status) self.assertEqual('1665695', job.job_ident) exp = [ 'sbatch', '--mem', '16000M', '-t', '16:00:00', '--export=ALL', '-J', 'job1', '-o', 'temp/job-%x-%A-%a.log', '--array=1-5,14,16%2', 'submit.sh', ] patch_check.assert_called_with(exp, shell=False)
def test_resubmission_array(self): content = """ JobID|JobIDRaw|JobName|Partition|MaxVMSize|MaxVMSizeNode|MaxVMSizeTask|AveVMSize|MaxRSS|MaxRSSNode|MaxRSSTask|AveRSS|MaxPages|MaxPagesNode|MaxPagesTask|AvePages|MinCPU|MinCPUNode|MinCPUTask|AveCPU|NTasks|AllocCPUS|Elapsed|State|ExitCode|AveCPUFreq|ReqCPUFreqMin|ReqCPUFreqMax|ReqCPUFreqGov|ReqMem|ConsumedEnergy|MaxDiskRead|MaxDiskReadNode|MaxDiskReadTask|AveDiskRead|MaxDiskWrite|MaxDiskWriteNode|MaxDiskWriteTask|AveDiskWrite|AllocGRES|ReqGRES|ReqTRES|AllocTRES 1873472_162|1873671|MV_P02300_batch-egprnnYFaJtPtnECYfGiKf|all||||||||||||||||||1|10:18:26|COMPLETED|0:0||Unknown|Unknown|Unknown|16000Mn||||||||||||cpu=1,mem=16000M,node=1|cpu=1,mem=16000M,node=1 1873472_162.batch|1873671.batch|batch||13703984K|n106|0|8708976K|11725204K|n106|0|6743424K|53K|n106|0|53K|10:06:31|n106|0|10:06:31|1|1|10:18:26|COMPLETED|0:0|2.19M|0|0|0|16000Mn|0|11767.09M|n106|0|11767.09M|29.74M|n106|0|29.74M||||cpu=1,mem=16000M,node=1 1873472_163|1873672|MV_P02300_batch-egprnnYFaJtPtnECYfGiKf|all||||||||||||||||||1|08:09:50|COMPLETED|0:0||Unknown|Unknown|Unknown|16000Mn||||||||||||cpu=1,mem=16000M,node=1|cpu=1,mem=16000M,node=1 1873472_163.batch|1873672.batch|batch||13690948K|n106|0|8686468K|11712556K|n106|0|6721328K|45K|n106|0|45K|07:57:40|n106|0|07:57:40|1|1|08:09:50|COMPLETED|0:0|2.19M|0|0|0|16000Mn|0|13345.62M|n106|0|13345.62M|26.77M|n106|0|26.77M||||cpu=1,mem=16000M,node=1 1873472_164|1873673|MV_P02300_batch-egprnnYFaJtPtnECYfGiKf|all||||||||||||||||||1|12:26:33|COMPLETED|0:0||Unknown|Unknown|Unknown|16000Mn||||||||||||cpu=1,mem=16000M,node=1|cpu=1,mem=16000M,node=1 1873472_164.batch|1873673.batch|batch||13730588K|n106|0|9577424K|11750552K|n106|0|6777084K|55K|n106|0|55K|12:13:52|n106|0|12:13:52|1|1|12:26:33|COMPLETED|0:0|2.19M|0|0|0|16000Mn|0|17065.30M|n106|0|17065.30M|34.39M|n106|0|34.39M||||cpu=1,mem=16000M,node=1 1873472_165|1873674|MV_P02300_batch-egprnnYFaJtPtnECYfGiKf|all||||||||||||||||||1|05:32:32|COMPLETED|0:0||Unknown|Unknown|Unknown|16000Mn||||||||||||cpu=1,mem=16000M,node=1|cpu=1,mem=16000M,node=1 1873472_165.batch|1873674.batch|batch||13735224K|n106|0|9574752K|11756988K|n106|0|6773916K|52K|n106|0|52K|05:21:46|n106|0|05:21:46|1|1|05:32:32|COMPLETED|0:0|2.18M|0|0|0|16000Mn|0|15997.17M|n106|0|15997.17M|37.74M|n106|0|37.74M||||cpu=1,mem=16000M,node=1 1873472_166|1873675|MV_P02300_batch-egprnnYFaJtPtnECYfGiKf|all||||||||||||||||||1|07:30:37|COMPLETED|0:0||Unknown|Unknown|Unknown|16000Mn||||||||||||cpu=1,mem=16000M,node=1|cpu=1,mem=16000M,node=1 1873472_166.batch|1873675.batch|batch||13722476K|n106|0|8669768K|11742400K|n106|0|6702776K|53K|n106|0|53K|07:18:31|n106|0|07:18:31|1|1|07:30:37|COMPLETED|0:0|2.19M|0|0|0|16000Mn|0|14716.82M|n106|0|14716.82M|21.39M|n106|0|21.39M||||cpu=1,mem=16000M,node=1 1873472_167|1873676|MV_P02300_batch-egprnnYFaJtPtnECYfGiKf|all||||||||||||||||||1|06:45:32|COMPLETED|0:0||Unknown|Unknown|Unknown|16000Mn||||||||||||cpu=1,mem=16000M,node=1|cpu=1,mem=16000M,node=1 1873472_167.batch|1873676.batch|batch||13686828K|n106|0|8596932K|11707132K|n106|0|6565180K|49K|n106|0|49K|06:35:26|n106|0|06:35:26|1|1|06:45:32|COMPLETED|0:0|2.19M|0|0|0|16000Mn|0|10274.82M|n106|0|10274.82M|39.37M|n106|0|39.37M||||cpu=1,mem=16000M,node=1 1873472_168|1873677|MV_P02300_batch-egprnnYFaJtPtnECYfGiKf|all||||||||||||||||||1|16:00:06|TIMEOUT|0:0||Unknown|Unknown|Unknown|16000Mn||||||||||||cpu=1,mem=16000M,node=1|cpu=1,mem=16000M,node=1 1873472_168.batch|1873677.batch|batch||13749848K|n106|0|8700272K|11771032K|n106|0|6734652K|46K|n106|0|46K|15:48:39|n106|0|15:48:39|1|1|16:00:07|CANCELLED|0:15|2.19M|0|0|0|16000Mn|0|10613.36M|n106|0|10613.36M|25.00M|n106|0|25.00M||||cpu=1,mem=16000M,node=1 """ rows = _scheduler.SlurmScheduler().parse_sacct(content) complete = [ row['status'] for row in rows if row['status'] == _constants.JOB_STATUS.COMPLETED ] fail = [row['status'] for row in rows if row['status'] == _constants.JOB_STATUS.CANCELLED] self.assertEqual(6, len(complete)) self.assertEqual(1, len(fail))
def test_partition(self, patch_check): patch_check.return_value = "Submitted batch job 1665695".encode('utf8') job = _job.Job( output_dir='temp', name='job1', stage='validate', script='submit.sh', queue='all' ) print(job) _scheduler.SlurmScheduler().submit(job) self.assertEqual(_constants.JOB_STATUS.SUBMITTED, job.status) self.assertEqual('1665695', job.job_ident) patch_check.assert_called_with( [ 'sbatch', '--partition=all', '--mem', '16000M', '-t', '16:00:00', '--export=ALL', '-J', 'job1', '-o', 'temp/job-%x-%j.log', 'submit.sh', ], shell=False, )
def test_bad_command(self, patcher): patcher.side_effect = [subprocess.CalledProcessError(1, 'cmd')] sched = _scheduler.SlurmScheduler() job = _job.Job(SUBCOMMAND.VALIDATE, '', job_ident='1234') with self.assertRaises(subprocess.CalledProcessError): sched.cancel(job) patcher.assert_called_with(['scancel', '1234'])
def test_dependent_job(self, patch_check): patch_check.side_effect = [ "Submitted batch job 1665695".encode('utf8') ] job = _job.Job(output_dir='temp', name='job1', stage='validate', script='submit.sh', dependencies=[ _job.Job(output_dir='temp', name='job2', stage='cluster', script='submit2.sh', job_ident='12345678') ]) print(job) _scheduler.SlurmScheduler().submit(job) self.assertEqual(_constants.JOB_STATUS.SUBMITTED, job.status) self.assertEqual('1665695', job.job_ident) patch_check.assert_called_with([ 'sbatch', '--mem', '16000M', '-t', '16:00:00', '--export=ALL', '--dependency=afterok:12345678', '-J', 'job1', '-o', 'temp/job-%x-%j.log', 'submit.sh' ], shell=False)
def test_array_job(self, patcher): sched = _scheduler.SlurmScheduler() job = _job.ArrayJob(SUBCOMMAND.VALIDATE, 10, output_dir='', job_ident='1234') sched.cancel(job) self.assertEqual(_constants.JOB_STATUS.CANCELLED, job.status) for task in job.task_list: self.assertEqual(_constants.JOB_STATUS.CANCELLED, task.status) patcher.assert_called_with(['scancel', '1234'])
def test_basic_table(self): content = """ JobID|JobIDRaw|JobName|Partition|MaxVMSize|MaxVMSizeNode|MaxVMSizeTask|AveVMSize|MaxRSS|MaxRSSNode|MaxRSSTask|AveRSS|MaxPages|MaxPagesNode|MaxPagesTask|AvePages|MinCPU|MinCPUNode|MinCPUTask|AveCPU|NTasks|AllocCPUS|Elapsed|State|ExitCode|AveCPUFreq|ReqCPUFreqMin|ReqCPUFreqMax|ReqCPUFreqGov|ReqMem|ConsumedEnergy|MaxDiskRead|MaxDiskReadNode|MaxDiskReadTask|AveDiskRead|MaxDiskWrite|MaxDiskWriteNode|MaxDiskWriteTask|AveDiskWrite|AllocGRES|ReqGRES|ReqTRES|AllocTRES 1672273|1672273|MS_batch-iJUMYRdLFDsuu9eVzGmmKm|short||||||||||||||||||1|00:00:00|CANCELLED by 1365|0:0||Unknown|Unknown|Unknown|16000Mn||||||||||||cpu=1,mem=16000M,node=1| """ rows = _scheduler.SlurmScheduler().parse_sacct(content) self.assertEqual(1, len(rows)) row = rows[0] self.assertEqual(_constants.JOB_STATUS.CANCELLED, row['status'])
def test_array_job_task(self, patcher): sched = _scheduler.SlurmScheduler() job = _job.ArrayJob(SUBCOMMAND.VALIDATE, 10, output_dir='', job_ident='1234') sched.cancel(job, task_ident=4) self.assertEqual(_constants.JOB_STATUS.NOT_SUBMITTED, job.status) for i, task in enumerate(job.task_list): if i == 3: self.assertEqual(_constants.JOB_STATUS.CANCELLED, task.status) else: self.assertEqual(_constants.JOB_STATUS.NOT_SUBMITTED, task.status) patcher.assert_called_with(['scancel', '1234_4'])
def test_pending_array(self): content = """ JobID|JobName|User|ReqMem|Elapsed|State|MaxRSS|AveRSS|Partition 1701003_[37-200]|MA_L1522785992-normal_batch-aUmErftiY7eEWvENfSeJwc|creisle|12000Mn|00:00:00|PENDING|||all 1701003_1|MA_L1522785992-normal_batch-aUmErftiY7eEWvENfSeJwc|creisle|12000Mn|00:05:00|RUNNING|||all """ rows = _scheduler.SlurmScheduler().parse_sacct(content) self.assertEqual(2, len(rows)) self.assertEqual(_constants.JOB_STATUS.PENDING, rows[0]['status']) self.assertEqual(_constants.JOB_STATUS.RUNNING, rows[1]['status']) self.assertIs(None, rows[0]['task_ident']) self.assertEqual(1, rows[1]['task_ident'])
def test_cancelled_task(self): content = """ JobID|JobName|User|ReqMem|Elapsed|State|MaxRSS|AveRSS|Partition 1697503_3|MV_mock-A47933_batch-uwSwW68EW43XNdvq85NxJ7|creisle|18000Mn|00:00:10|COMPLETED|||all 1697503_3.batch|batch||18000Mn|00:00:10|COMPLETED|904K|904K| 1697503_1|MV_mock-A47933_batch-uwSwW68EW43XNdvq85NxJ7|creisle|18000Mn|00:00:02|CANCELLED by 1365|||all 1697503_1.batch|batch||18000Mn|00:00:02|CANCELLED|896K|896K| 1697503_2|MV_mock-A47933_batch-uwSwW68EW43XNdvq85NxJ7|creisle|18000Mn|00:00:10|COMPLETED|||all 1697503_2.batch|batch||18000Mn|00:00:10|COMPLETED|904K|904K| """ rows = _scheduler.SlurmScheduler().parse_sacct(content) self.assertEqual(3, len(rows)) self.assertEqual(_constants.JOB_STATUS.CANCELLED, rows[1]['status']) self.assertEqual(_constants.JOB_STATUS.COMPLETED, rows[0]['status'])
def test_dependency_error(self, patch_check): patch_check.side_effect = [ "Submitted batch job 12345678".encode('utf8'), "Submitted batch job 1665695".encode('utf8'), ] job = _job.Job( output_dir='temp', name='job1', stage='validate', script='submit.sh', dependencies=[ _job.Job(output_dir='temp', name='job2', stage='cluster', script='submit2.sh') ], ) print(job) with self.assertRaises(ValueError): _scheduler.SlurmScheduler().submit(job)
def test_pending_job(self): content = """ JobId=1673292 JobName=MP_batch-8PyNX8EN4cBdD9vQd9FrRG UserId=creisle(1365) GroupId=users(100) MCS_label=N/A Priority=31 Nice=0 Account=all QOS=normal JobState=PENDING Reason=DependencyNeverSatisfied Dependency=afterok:1673291_* Requeue=1 Restarts=0 BatchFlag=1 Reboot=0 ExitCode=0:0 RunTime=00:00:00 TimeLimit=16:00:00 TimeMin=N/A SubmitTime=2018-05-24T11:32:44 EligibleTime=Unknown StartTime=Unknown EndTime=Unknown Deadline=N/A PreemptTime=None SuspendTime=None SecsPreSuspend=0 Partition=short AllocNode:Sid=n104:47409 ReqNodeList=(null) ExcNodeList=(null) NodeList=(null) NumNodes=1 NumCPUs=1 NumTasks=1 CPUs/Task=1 ReqB:S:C:T=0:0:*:* TRES=cpu=1,mem=16000,node=1 Socks/Node=* NtasksPerN:B:S:C=0:0:*:* CoreSpec=* MinCPUsNode=1 MinMemoryNode=16000M MinTmpDiskNode=0 Features=(null) DelayBoot=00:00:00 Gres=(null) Reservation=(null) OverSubscribe=OK Contiguous=0 Licenses=(null) Network=(null) Command=/projects/trans_scratch/validations/workspace/creisle/temp/test_submission/output_slurm/pairing/submit.sh WorkDir=/projects/trans_scratch/validations/workspace/creisle/temp/test_submission StdErr=/projects/trans_scratch/validations/workspace/creisle/temp/test_submission/output_slurm/pairing/job-%x-1673292.log StdIn=/dev/null StdOut=/projects/trans_scratch/validations/workspace/creisle/temp/test_submission/output_slurm/pairing/job-%x-1673292.log Power= """ rows = _scheduler.SlurmScheduler().parse_scontrol_show(content) self.assertEqual(1, len(rows)) self.assertEqual( { 'job_ident': '1673292', 'task_ident': None, 'status': 'PENDING', 'status_comment': 'DependencyNeverSatisfied', 'name': 'MP_batch-8PyNX8EN4cBdD9vQd9FrRG', }, rows[0], )
def test_job_array(self, patch_check): content = """ JobID|JobIDRaw|JobName|Partition|MaxVMSize|MaxVMSizeNode|MaxVMSizeTask|AveVMSize|MaxRSS|MaxRSSNode|MaxRSSTask|AveRSS|MaxPages|MaxPagesNode|MaxPagesTask|AvePages|MinCPU|MinCPUNode|MinCPUTask|AveCPU|NTasks|AllocCPUS|Elapsed|State|ExitCode|AveCPUFreq|ReqCPUFreqMin|ReqCPUFreqMax|ReqCPUFreqGov|ReqMem|ConsumedEnergy|MaxDiskRead|MaxDiskReadNode|MaxDiskReadTask|AveDiskRead|MaxDiskWrite|MaxDiskWriteNode|MaxDiskWriteTask|AveDiskWrite|AllocGRES|ReqGRES|ReqTRES|AllocTRES| 1671879_1|1671879|MV_mock-A36971_batch-tX8SW6tEiEfZ8ZLHDPDa83|short||||||||||||||||||1|00:00:00|FAILED|1:0||Unknown|Unknown|Unknown|16000Mn||||||||||||cpu=1,mem=16000M,node=1|cpu=1,mem=16000M,node=1| 1671879_1.batch|1671879.batch|batch||||||||||||||||||1|1|00:00:00|FAILED|1:0||0|0|0|16000Mn|||||||||||||cpu=1,mem=16000M,node=1| 1671880_1|1671880|MV_mock-A47933_batch-tX8SW6tEiEfZ8ZLHDPDa83|short||||||||||||||||||1|00:00:00|FAILED|1:0||Unknown|Unknown|Unknown|18000Mn||||||||||||cpu=1,mem=18000M,node=1|cpu=1,mem=18000M,node=1| 1671880_1.batch|1671880.batch|batch||||||||||||||||||1|1|00:00:00|FAILED|1:0||0|0|0|18000Mn|||||||||||||cpu=1,mem=18000M,node=1| 1671893_1|1671893|MV_mock-A36971_batch-tX8SW6tEiEfZ8ZLHDPDa83|short||||||||||||||||||1|00:00:01|FAILED|1:0||Unknown|Unknown|Unknown|16000Mn||||||||||||cpu=1,mem=16000M,node=1|cpu=1,mem=16000M,node=1| 1671893_1.batch|1671893.batch|batch||||||||||||||||||1|1|00:00:01|FAILED|1:0||0|0|0|16000Mn|||||||||||||cpu=1,mem=16000M,node=1| 1671894_1|1671894|MV_mock-A47933_batch-tX8SW6tEiEfZ8ZLHDPDa83|short||||||||||||||||||1|00:00:00|FAILED|1:0||Unknown|Unknown|Unknown|18000Mn||||||||||||cpu=1,mem=18000M,node=1|cpu=1,mem=18000M,node=1| 1671894_1.batch|1671894.batch|batch||||||||||||||||||1|1|00:00:00|FAILED|1:0||0|0|0|18000Mn|||||||||||||cpu=1,mem=18000M,node=1| 1671915_1|1671915|MV_mock-A36971_batch-ezPmnHmYjZjsj8gfCynbsX|short||||||||||||||||||1|00:00:20|CANCELLED by 1365|0:0||Unknown|Unknown|Unknown|16000Mn||||||||||||cpu=1,mem=16000M,node=1|cpu=1,mem=16000M,node=1| 1671915_1.batch|1671915.batch|batch||125588K|n305|0|125588K|900K|n305|0|900K|0|n305|0|0|00:00:00|n305|0|00:00:00|1|1|00:00:20|CANCELLED|0:15|2.19M|0|0|0|16000Mn|0|0|n305|65534|0|0|n305|65534|0||||cpu=1,mem=16000M,node=1| 1671916_1|1671916|MV_mock-A47933_batch-ezPmnHmYjZjsj8gfCynbsX|short||||||||||||||||||1|00:00:20|CANCELLED by 1365|0:0||Unknown|Unknown|Unknown|18000Mn||||||||||||cpu=1,mem=18000M,node=1|cpu=1,mem=18000M,node=1| 1671916_1.batch|1671916.batch|batch||125588K|n305|0|125588K|900K|n305|0|900K|0|n305|0|0|00:00:00|n305|0|00:00:00|1|1|00:00:20|CANCELLED|0:15|2.19M|0|0|0|18000Mn|0|0|n305|65534|0|0|n305|65534|0||||cpu=1,mem=18000M,node=1| 1671970_1|1671970|MV_mock-A36971_batch-ezPmnHmYjZjsj8gfCynbsX|short||||||||||||||||||1|00:00:21|COMPLETED|0:0||Unknown|Unknown|Unknown|16000Mn||||||||||||cpu=1,mem=16000M,node=1|cpu=1,mem=16000M,node=1| 1671970_1.batch|1671970.batch|batch||125588K|n305|0|125588K|908K|n305|0|908K|0|n305|0|0|00:00:00|n305|0|00:00:00|1|1|00:00:21|COMPLETED|0:0|2.19M|0|0|0|16000Mn|0|0|n305|65534|0|0|n305|65534|0||||cpu=1,mem=16000M,node=1| 1671971_1|1671971|MV_mock-A47933_batch-ezPmnHmYjZjsj8gfCynbsX|short||||||||||||||||||1|00:00:20|COMPLETED|0:0||Unknown|Unknown|Unknown|18000Mn||||||||||||cpu=1,mem=18000M,node=1|cpu=1,mem=18000M,node=1| 1671971_1.batch|1671971.batch|batch||125588K|n305|0|125588K|904K|n305|0|904K|0|n305|0|0|00:00:00|n305|0|00:00:00|1|1|00:00:20|COMPLETED|0:0|2.19M|0|0|0|18000Mn|0|0|n305|65534|0|0|n305|65534|0||||cpu=1,mem=18000M,node=1| 1671974_1|1671974|MV_mock-A36971_batch-ezPmnHmYjZjsj8gfCynbsX|short||||||||||||||||||1|00:00:11|COMPLETED|0:0||Unknown|Unknown|Unknown|16000Mn||||||||||||cpu=1,mem=16000M,node=1|cpu=1,mem=16000M,node=1| 1671974_1.batch|1671974.batch|batch||125588K|n305|0|125588K|900K|n305|0|900K|0|n305|0|0|00:00:00|n305|0|00:00:00|1|1|00:00:11|COMPLETED|0:0|2.19M|0|0|0|16000Mn|0|0|n305|65534|0|0|n305|65534|0||||cpu=1,mem=16000M,node=1| 1671975_1|1671975|MV_mock-A47933_batch-ezPmnHmYjZjsj8gfCynbsX|short||||||||||||||||||1|00:00:10|COMPLETED|0:0||Unknown|Unknown|Unknown|18000Mn||||||||||||cpu=1,mem=18000M,node=1|cpu=1,mem=18000M,node=1| 1671975_1.batch|1671975.batch|batch||125588K|n305|0|125588K|908K|n305|0|908K|0|n305|0|0|00:00:00|n305|0|00:00:00|1|1|00:00:10|COMPLETED|0:0|2.19M|0|0|0|18000Mn|0|0|n305|65534|0|0|n305|65534|0||||cpu=1,mem=18000M,node=1| 1671981_1|1671981|MV_mock-A36971_batch-ezPmnHmYjZjsj8gfCynbsX|short||||||||||||||||||1|00:00:12|COMPLETED|0:0||Unknown|Unknown|Unknown|16000Mn||||||||||||cpu=1,mem=16000M,node=1|cpu=1,mem=16000M,node=1| 1671981_1.batch|1671981.batch|batch||125588K|n305|0|125588K|904K|n305|0|904K|0|n305|0|0|00:00:00|n305|0|00:00:00|1|1|00:00:12|COMPLETED|0:0|2.19M|0|0|0|16000Mn|0|0|n305|65534|0|0|n305|65534|0||||cpu=1,mem=16000M,node=1| 1671982_1|1671982|MV_mock-A47933_batch-ezPmnHmYjZjsj8gfCynbsX|short||||||||||||||||||1|00:00:11|COMPLETED|0:0||Unknown|Unknown|Unknown|18000Mn||||||||||||cpu=1,mem=18000M,node=1|cpu=1,mem=18000M,node=1| 1671982_1.batch|1671982.batch|batch||125588K|n305|0|125588K|900K|n305|0|900K|0|n305|0|0|00:00:00|n305|0|00:00:00|1|1|00:00:11|COMPLETED|0:0|2.19M|0|0|0|18000Mn|0|0|n305|65534|0|0|n305|65534|0||||cpu=1,mem=18000M,node=1| 1671983_1|1671983|MA_mock-A36971_batch-ezPmnHmYjZjsj8gfCynbsX|short||||||||||||||||||1|00:00:05|COMPLETED|0:0||Unknown|Unknown|Unknown|12000Mn||||||||||||cpu=1,mem=12000M,node=1|cpu=1,mem=12000M,node=1| 1671983_1.batch|1671983.batch|batch||125588K|n305|0|125588K|896K|n305|0|896K|0|n305|0|0|00:00:00|n305|0|00:00:00|1|1|00:00:05|COMPLETED|0:0|2.19M|0|0|0|12000Mn|0|0|n305|65534|0|0|n305|65534|0||||cpu=1,mem=12000M,node=1| 1671984_1|1671984|MA_mock-A47933_batch-ezPmnHmYjZjsj8gfCynbsX|short||||||||||||||||||1|00:00:04|COMPLETED|0:0||Unknown|Unknown|Unknown|12000Mn||||||||||||cpu=1,mem=12000M,node=1|cpu=1,mem=12000M,node=1| 1671984_1.batch|1671984.batch|batch||125588K|n305|0|125588K|896K|n305|0|896K|0|n305|0|0|00:00:00|n305|0|00:00:00|1|1|00:00:04|COMPLETED|0:0|2.19M|0|0|0|12000Mn|0|0|n305|65534|0|0|n305|65534|0||||cpu=1,mem=12000M,node=1| 1671985|1671985|MP_batch-ezPmnHmYjZjsj8gfCynbsX|short||||||||||||||||||1|00:00:02|FAILED|2:0||Unknown|Unknown|Unknown|16000Mn||||||||||||cpu=1,mem=16000M,node=1|cpu=1,mem=16000M,node=1| 1671985.batch|1671985.batch|batch||125588K|n305|0|125588K|900K|n305|0|900K|0|n305|0|0|00:00:00|n305|0|00:00:00|1|1|00:00:02|FAILED|2:0|2.19M|0|0|0|16000Mn|0|0|n305|65534|0|0|n305|65534|0||||cpu=1,mem=16000M,node=1| 1671986|1671986|MS_batch-ezPmnHmYjZjsj8gfCynbsX|short||||||||||||||||||1|00:00:00|CANCELLED by 1365|0:0||Unknown|Unknown|Unknown|16000Mn||||||||||||cpu=1,mem=16000M,node=1|| 1672141|1672141|subtest.sh|all||||||||||||||||||1|00:01:01|COMPLETED|0:0||Unknown|Unknown|Unknown|7000Mc||||||||||||cpu=1,mem=7000M,node=1|cpu=1,mem=7000M,node=1| 1672141.batch|1672141.batch|batch||207004K|n106|0|207004K|1760K|n106|0|1760K|1K|n106|0|1K|00:00:00|n106|0|00:00:00|1|1|00:01:01|COMPLETED|0:0|2.19M|0|0|0|7000Mc|0|0.06M|n106|0|0.06M|0.00M|n106|0|0.00M||||cpu=1,mem=7000M,node=1| 1672166|1672166|subtest.sh|all||||||||||||||||||1|00:01:03|COMPLETED|0:0||Unknown|Unknown|Unknown|7000Mc||||||||||||cpu=1,mem=7000M,node=1|cpu=1,mem=7000M,node=1| 1672166.batch|1672166.batch|batch||207004K|n106|0|207004K|1760K|n106|0|1760K|0|n106|0|0|00:00:00|n106|0|00:00:00|1|1|00:01:03|COMPLETED|0:0|2.19M|0|0|0|7000Mc|0|0.06M|n106|0|0.06M|0.00M|n106|0|0.00M||||cpu=1,mem=7000M,node=1| 1672169_10|1672169|subtest.sh|all||||||||||||||||||1|00:01:02|COMPLETED|0:0||Unknown|Unknown|Unknown|7000Mc||||||||||||cpu=1,mem=7000M,node=1|cpu=1,mem=7000M,node=1| 1672169_10.batch|1672169.batch|batch||207004K|n130|0|207004K|1764K|n130|0|1764K|0|n130|0|0|00:00:00|n130|0|00:00:00|1|1|00:01:02|COMPLETED|0:0|2.19M|0|0|0|7000Mc|0|0.06M|n130|0|0.06M|0.00M|n130|0|0.00M||||cpu=1,mem=7000M,node=1| 1672169_1|1672171|subtest.sh|all||||||||||||||||||1|00:01:00|COMPLETED|0:0||Unknown|Unknown|Unknown|7000Mc||||||||||||cpu=1,mem=7000M,node=1|cpu=1,mem=7000M,node=1| 1672169_1.batch|1672171.batch|batch||207004K|n106|0|207004K|1764K|n106|0|1764K|0|n106|0|0|00:00:00|n106|0|00:00:00|1|1|00:01:00|COMPLETED|0:0|2.19M|0|0|0|7000Mc|0|0.06M|n106|0|0.06M|0.00M|n106|0|0.00M||||cpu=1,mem=7000M,node=1| 1672169_2|1672172|subtest.sh|all||||||||||||||||||1|00:01:00|COMPLETED|0:0||Unknown|Unknown|Unknown|7000Mc||||||||||||cpu=1,mem=7000M,node=1|cpu=1,mem=7000M,node=1| 1672169_2.batch|1672172.batch|batch||207004K|n106|0|207004K|1764K|n106|0|1764K|0|n106|0|0|00:00:00|n106|0|00:00:00|1|1|00:01:00|COMPLETED|0:0|2.19M|0|0|0|7000Mc|0|0.06M|n106|0|0.06M|0.00M|n106|0|0.00M||||cpu=1,mem=7000M,node=1| 1672169_3|1672173|subtest.sh|all||||||||||||||||||1|00:01:01|COMPLETED|0:0||Unknown|Unknown|Unknown|7000Mc||||||||||||cpu=1,mem=7000M,node=1|cpu=1,mem=7000M,node=1| 1672169_3.batch|1672173.batch|batch||207004K|n106|0|207004K|1764K|n106|0|1764K|0|n106|0|0|00:00:00|n106|0|00:00:00|1|1|00:01:01|COMPLETED|0:0|2.19M|0|0|0|7000Mc|0|0.06M|n106|0|0.06M|0.00M|n106|0|0.00M||||cpu=1,mem=7000M,node=1| 1672169_4|1672174|subtest.sh|all||||||||||||||||||1|00:01:01|COMPLETED|0:0||Unknown|Unknown|Unknown|7000Mc||||||||||||cpu=1,mem=7000M,node=1|cpu=1,mem=7000M,node=1| 1672169_4.batch|1672174.batch|batch||207004K|n106|0|207004K|1760K|n106|0|1760K|0|n106|0|0|00:00:00|n106|0|00:00:00|1|1|00:01:01|COMPLETED|0:0|2.19M|0|0|0|7000Mc|0|0.06M|n106|0|0.06M|0.00M|n106|0|0.00M||||cpu=1,mem=7000M,node=1| 1672169_5|1672175|subtest.sh|all||||||||||||||||||1|00:01:02|COMPLETED|0:0||Unknown|Unknown|Unknown|7000Mc||||||||||||cpu=1,mem=7000M,node=1|cpu=1,mem=7000M,node=1| 1672169_5.batch|1672175.batch|batch||207004K|n130|0|207004K|1764K|n130|0|1764K|0|n130|0|0|00:00:00|n130|0|00:00:00|1|1|00:01:02|COMPLETED|0:0|2.19M|0|0|0|7000Mc|0|0.06M|n130|0|0.06M|0.00M|n130|0|0.00M||||cpu=1,mem=7000M,node=1| 1672169_6|1672176|subtest.sh|all||||||||||||||||||1|00:01:02|COMPLETED|0:0||Unknown|Unknown|Unknown|7000Mc||||||||||||cpu=1,mem=7000M,node=1|cpu=1,mem=7000M,node=1| 1672169_6.batch|1672176.batch|batch||207004K|n130|0|207004K|1764K|n130|0|1764K|1K|n130|0|1K|00:00:00|n130|0|00:00:00|1|1|00:01:02|COMPLETED|0:0|2.19M|0|0|0|7000Mc|0|0.06M|n130|0|0.06M|0.00M|n130|0|0.00M||||cpu=1,mem=7000M,node=1| 1672169_7|1672177|subtest.sh|all||||||||||||||||||1|00:01:01|COMPLETED|0:0||Unknown|Unknown|Unknown|7000Mc||||||||||||cpu=1,mem=7000M,node=1|cpu=1,mem=7000M,node=1| 1672169_7.batch|1672177.batch|batch||207004K|n130|0|207004K|1756K|n130|0|1756K|1K|n130|0|1K|00:00:00|n130|0|00:00:00|1|1|00:01:01|COMPLETED|0:0|2.19M|0|0|0|7000Mc|0|0.06M|n130|0|0.06M|0.00M|n130|0|0.00M||||cpu=1,mem=7000M,node=1| 1672169_8|1672178|subtest.sh|all||||||||||||||||||1|00:01:02|COMPLETED|0:0||Unknown|Unknown|Unknown|7000Mc||||||||||||cpu=1,mem=7000M,node=1|cpu=1,mem=7000M,node=1| 1672169_8.batch|1672178.batch|batch||207004K|n130|0|207004K|1764K|n130|0|1764K|0|n130|0|0|00:00:00|n130|0|00:00:00|1|1|00:01:02|COMPLETED|0:0|2.19M|0|0|0|7000Mc|0|0.06M|n130|0|0.06M|0.00M|n130|0|0.00M||||cpu=1,mem=7000M,node=1| 1672169_9|1672179|subtest.sh|all||||||||||||||||||1|00:01:02|COMPLETED|0:0||Unknown|Unknown|Unknown|7000Mc||||||||||||cpu=1,mem=7000M,node=1|cpu=1,mem=7000M,node=1| 1672169_9.batch|1672179.batch|batch||207004K|n130|0|207004K|1760K|n130|0|1760K|1K|n130|0|1K|00:00:00|n130|0|00:00:00|1|1|00:01:02|COMPLETED|0:0|2.19M|0|0|0|7000Mc|0|0.06M|n130|0|0.06M|0.00M|n130|0|0.00M||||cpu=1,mem=7000M,node=1| 1672268_2|1672268|MV_mock-A36971_batch-iJUMYRdLFDsuu9eVzGmmKm|short||||||||||||||||||1|00:00:36|COMPLETED|0:0||Unknown|Unknown|Unknown|16000Mn||||||||||||cpu=1,mem=16000M,node=1|cpu=1,mem=16000M,node=1| 1672268_2.batch|1672268.batch|batch||2025280K|n305|0|2025280K|58776K|n305|0|58776K|0|n305|0|0|00:00:03|n305|0|00:00:03|1|1|00:00:36|COMPLETED|0:0|2.19M|0|0|0|16000Mn|0|13.01M|n305|0|13.01M|0.05M|n305|0|0.05M||||cpu=1,mem=16000M,node=1| 1672269_3|1672269|MV_mock-A47933_batch-iJUMYRdLFDsuu9eVzGmmKm|short||||||||||||||||||1|00:00:33|COMPLETED|0:0||Unknown|Unknown|Unknown|18000Mn||||||||||||cpu=1,mem=18000M,node=1|cpu=1,mem=18000M,node=1| 1672269_3.batch|1672269.batch|batch||2019424K|n305|0|2019424K|55844K|n305|0|55844K|0|n305|0|0|00:00:03|n305|0|00:00:03|1|1|00:00:33|COMPLETED|0:0|2.19M|0|0|0|18000Mn|0|11.60M|n305|0|11.60M|0.05M|n305|0|0.05M||||cpu=1,mem=18000M,node=1| 1672270_2|1672270|MA_mock-A36971_batch-iJUMYRdLFDsuu9eVzGmmKm|short||||||||||||||||||1|00:00:02|COMPLETED|0:0||Unknown|Unknown|Unknown|12000Mn||||||||||||cpu=1,mem=12000M,node=1|cpu=1,mem=12000M,node=1| 1672270_2.batch|1672270.batch|batch||125588K|n305|0|125588K|904K|n305|0|904K|0|n305|0|0|00:00:00|n305|0|00:00:00|1|1|00:00:02|COMPLETED|0:0|2.19M|0|0|0|12000Mn|0|0|n305|65534|0|0|n305|65534|0||||cpu=1,mem=12000M,node=1| 1672271_3|1672271|MA_mock-A47933_batch-iJUMYRdLFDsuu9eVzGmmKm|short||||||||||||||||||1|00:00:02|COMPLETED|0:0||Unknown|Unknown|Unknown|12000Mn||||||||||||cpu=1,mem=12000M,node=1|cpu=1,mem=12000M,node=1| 1672271_3.batch|1672271.batch|batch||125588K|n305|0|125588K|904K|n305|0|904K|0|n305|0|0|00:00:00|n305|0|00:00:00|1|1|00:00:02|COMPLETED|0:0|2.19M|0|0|0|12000Mn|0|0|n305|65534|0|0|n305|65534|0||||cpu=1,mem=12000M,node=1| 1672272|1672272|MP_batch-iJUMYRdLFDsuu9eVzGmmKm|short||||||||||||||||||1|00:00:02|FAILED|2:0||Unknown|Unknown|Unknown|16000Mn||||||||||||cpu=1,mem=16000M,node=1|cpu=1,mem=16000M,node=1| 1672272.batch|1672272.batch|batch||125588K|n305|0|125588K|900K|n305|0|900K|0|n305|0|0|00:00:00|n305|0|00:00:00|1|1|00:00:02|FAILED|2:0|2.19M|0|0|0|16000Mn|0|0|n305|65534|0|0|n305|65534|0||||cpu=1,mem=16000M,node=1| 1672273|1672273|MS_batch-iJUMYRdLFDsuu9eVzGmmKm|short||||||||||||||||||1|00:00:00|CANCELLED by 1365|0:0||Unknown|Unknown|Unknown|16000Mn||||||||||||cpu=1,mem=16000M,node=1|| 1672270_1|1672274|MA_mock-A36971_batch-iJUMYRdLFDsuu9eVzGmmKm|short||||||||||||||||||1|00:00:04|COMPLETED|0:0||Unknown|Unknown|Unknown|12000Mn||||||||||||cpu=1,mem=12000M,node=1|cpu=1,mem=12000M,node=1| 1672270_1.batch|1672274.batch|batch||125588K|n305|0|125588K|896K|n305|0|896K|0|n305|0|0|00:00:00|n305|0|00:00:00|1|1|00:00:04|COMPLETED|0:0|2.19M|0|0|0|12000Mn|0|0|n305|65534|0|0|n305|65534|0||||cpu=1,mem=12000M,node=1| 1672271_1|1672275|MA_mock-A47933_batch-iJUMYRdLFDsuu9eVzGmmKm|short||||||||||||||||||1|00:00:02|COMPLETED|0:0||Unknown|Unknown|Unknown|12000Mn||||||||||||cpu=1,mem=12000M,node=1|cpu=1,mem=12000M,node=1| 1672271_1.batch|1672275.batch|batch||125588K|n305|0|125588K|900K|n305|0|900K|0|n305|0|0|00:00:00|n305|0|00:00:00|1|1|00:00:02|COMPLETED|0:0|2.19M|0|0|0|12000Mn|0|0|n305|65534|0|0|n305|65534|0||||cpu=1,mem=12000M,node=1| 1672271_2|1672276|MA_mock-A47933_batch-iJUMYRdLFDsuu9eVzGmmKm|short||||||||||||||||||1|00:00:02|COMPLETED|0:0||Unknown|Unknown|Unknown|12000Mn||||||||||||cpu=1,mem=12000M,node=1|cpu=1,mem=12000M,node=1| 1672271_2.batch|1672276.batch|batch||125588K|n305|0|125588K|896K|n305|0|896K|0|n305|0|0|00:00:00|n305|0|00:00:00|1|1|00:00:02|COMPLETED|0:0|2.19M|0|0|0|12000Mn|0|0|n305|65534|0|0|n305|65534|0||||cpu=1,mem=12000M,node=1| 1672268_1|1672277|MV_mock-A36971_batch-iJUMYRdLFDsuu9eVzGmmKm|short||||||||||||||||||1|00:00:32|COMPLETED|0:0||Unknown|Unknown|Unknown|16000Mn||||||||||||cpu=1,mem=16000M,node=1|cpu=1,mem=16000M,node=1| 1672268_1.batch|1672277.batch|batch||2018912K|n305|0|2018912K|51952K|n305|0|51952K|0|n305|0|0|00:00:03|n305|0|00:00:03|1|1|00:00:32|COMPLETED|0:0|2.19M|0|0|0|16000Mn|0|10.79M|n305|0|10.79M|0.05M|n305|0|0.05M||||cpu=1,mem=16000M,node=1| 1672269_1|1672278|MV_mock-A47933_batch-iJUMYRdLFDsuu9eVzGmmKm|short||||||||||||||||||1|00:00:32|COMPLETED|0:0||Unknown|Unknown|Unknown|18000Mn||||||||||||cpu=1,mem=18000M,node=1|cpu=1,mem=18000M,node=1| 1672269_1.batch|1672278.batch|batch||2018660K|n305|0|2018660K|57016K|n305|0|57016K|0|n305|0|0|00:00:03|n305|0|00:00:03|1|1|00:00:32|COMPLETED|0:0|2.19M|0|0|0|18000Mn|0|12.79M|n305|0|12.79M|0.05M|n305|0|0.05M||||cpu=1,mem=18000M,node=1| 1672269_2|1672279|M1673291_mock-A47933_batch-iJUMYRdLFDsuu9eVzGmmKm|short||||||||||||||||||1|00:00:33|COMPLETED|0:0||Unknown|Unknown|Unknown|18000Mn||||||||||||cpu=1,mem=18000M,node=1|cpu=1,mem=18000M,node=1| 1672269_2.batch|1671673291279.batch|batch||2019424K|n305|0|2019424K|54212K|n305|0|54212K|0|n305|0|0|00:00:03|n305|0|00:00:03|1|1|00:00:33|COMPLETED|0:0|2.19M|0|0|0|18000Mn|0|11.51M|n305|0|11.51M|0.05M|n305|0|0.05M||||cpu=1,mem=18000M,node=1| 1672454_2|1672454|M1673291_mock-A36971_batch-uKEUyUuWbi2mgd75KjP4k5|short||||||||||||||||||1|00:00:29|COMPLETED|0:0||Unknown|Unknown|Unknown|16000Mn||||||||||||cpu=1,mem=16000M,node=1|cpu=1,mem=16000M,node=1| 1672454_2.batch|1671673291454.batch|batch||125588K|n305|0|125588K|908K|n305|0|908K|0|n305|0|0|00:00:00|n305|0|00:00:00|1|1|00:00:29|COMPLETED|0:0|2.19M|0|0|0|16000Mn|0|0|n305|65534|0|0|n305|65534|0||||cpu=1,mem=16000M,node=1| 1672455_3|1672455|MV_mock-A47933_batch-uKEUyUuWbi2mgd75KjP4k5|short||||||||||||||||||1|00:00:26|COMPLETED|0:0||Unknown|Unknown|Unknown|18000Mn||||||||||||cpu=1,mem=18000M,node=1|cpu=1,mem=18000M,node=1| 1672455_3.batch|1672455.batch|batch||125588K|n305|0|125588K|904K|n305|0|904K|0|n305|0|0|00:00:00|n305|0|00:00:00|1|1|00:00:26|COMPLETED|0:0|2.19M|0|0|0|18000Mn|0|0|n305|65534|0|0|n305|65534|0||||cpu=1,mem=18000M,node=1| 1672456_2|1672456|MA_mock-A36971_batch-uKEUyUuWbi2mgd75KjP4k5|short||||||||||||||||||1|00:00:02|COMPLETED|0:0||Unknown|Unknown|Unknown|12000Mn||||||||||||cpu=1,mem=12000M,node=1|cpu=1,mem=12000M,node=1| 1672456_2.batch|1672456.batch|batch||125588K|n305|0|125588K|904K|n305|0|904K|0|n305|0|0|00:00:00|n305|0|00:00:00|1|1|00:00:02|COMPLETED|0:0|2.19M|0|0|0|12000Mn|0|0|n305|65534|0|0|n305|65534|0||||cpu=1,mem=12000M,node=1| 1672457_3|1672457|MA_mock-A47933_batch-uKEUyUuWbi2mgd75KjP4k5|short||||||||||||||||||1|00:00:03|COMPLETED|0:0||Unknown|Unknown|Unknown|12000Mn||||||||||||cpu=1,mem=12000M,node=1|cpu=1,mem=12000M,node=1| 1672457_3.batch|1672457.batch|batch||125588K|n305|0|125588K|900K|n305|0|900K|0|n305|0|0|00:00:00|n305|0|00:00:00|1|1|00:00:03|COMPLETED|0:0|2.19M|0|0|0|12000Mn|0|0|n305|65534|0|0|n305|65534|0||||cpu=1,mem=12000M,node=1| 1672458|1672458|MP_batch-uKEUyUuWbi2mgd75KjP4k5|short||||||||||||||||||1|00:00:02|COMPLETED|0:0||Unknown|Unknown|Unknown|16000Mn||||||||||||cpu=1,mem=16000M,node=1|cpu=1,mem=16000M,node=1| 1672458.batch|1672458.batch|batch||125588K|n305|0|125588K|896K|n305|0|896K|0|n305|0|0|00:00:00|n305|0|00:00:00|1|1|00:00:02|COMPLETED|0:0|2.19M|0|0|0|16000Mn|0|0|n305|65534|0|0|n305|65534|0||||cpu=1,mem=16000M,node=1| 1672456_1|1672459|MA_mock-A36971_batch-uKEUyUuWbi2mgd75KjP4k5|short||||||||||||||||||1|00:00:04|COMPLETED|0:0||Unknown|Unknown|Unknown|12000Mn||||||||||||cpu=1,mem=12000M,node=1|cpu=1,mem=12000M,node=1| 1672456_1.batch|1672459.batch|batch||125588K|n305|0|125588K|900K|n305|0|900K|0|n305|0|0|00:00:00|n305|0|00:00:00|1|1|00:00:04|COMPLETED|0:0|2.19M|0|0|0|12000Mn|0|0|n305|65534|0|0|n305|65534|0||||cpu=1,mem=12000M,node=1| 1672457_1|1672460|MA_mock-A47933_batch-uKEUyUuWbi2mgd75KjP4k5|short||||||||||||||||||1|00:00:03|COMPLETED|0:0||Unknown|Unknown|Unknown|12000Mn||||||||||||cpu=1,mem=12000M,node=1|cpu=1,mem=12000M,node=1| 1672457_1.batch|1672460.batch|batch||125588K|n305|0|125588K|900K|n305|0|900K|0|n305|0|0|00:00:00|n305|0|00:00:00|1|1|00:00:03|COMPLETED|0:0|2.19M|0|0|0|12000Mn|0|0|n305|65534|0|0|n305|65534|0||||cpu=1,mem=12000M,node=1| 1672457_2|1672461|MA_mock-A47933_batch-uKEUyUuWbi2mgd75KjP4k5|short||||||||||||||||||1|00:00:02|COMPLETED|0:0||Unknown|Unknown|Unknown|12000Mn||||||||||||cpu=1,mem=12000M,node=1|cpu=1,mem=12000M,node=1| 1672457_2.batch|1672461.batch|batch||125588K|n305|0|125588K|896K|n305|0|896K|0|n305|0|0|00:00:00|n305|0|00:00:00|1|1|00:00:02|COMPLETED|0:0|2.19M|0|0|0|12000Mn|0|0|n305|65534|0|0|n305|65534|0||||cpu=1,mem=12000M,node=1| 1672454_1|1672462|MV_mock-A36971_batch-uKEUyUuWbi2mgd75KjP4k5|short||||||||||||||||||1|00:00:25|COMPLETED|0:0||Unknown|Unknown|Unknown|16000Mn||||||||||||cpu=1,mem=16000M,node=1|cpu=1,mem=16000M,node=1| 1672454_1.batch|1672462.batch|batch||125588K|n305|0|125588K|904K|n305|0|904K|0|n305|0|0|00:00:00|n305|0|00:00:00|1|1|00:00:25|COMPLETED|0:0|2.19M|0|0|0|16000Mn|0|0|n305|65534|0|0|n305|65534|0||||cpu=1,mem=16000M,node=1| 1672455_1|1672463|MV_mock-A47933_batch-uKEUyUuWbi2mgd75KjP4k5|short||||||||||||||||||1|00:00:25|COMPLETED|0:0||Unknown|Unknown|Unknown|18000Mn||||||||||||cpu=1,mem=18000M,node=1|cpu=1,mem=18000M,node=1| 1672455_1.batch|1672463.batch|batch||125588K|n305|0|125588K|904K|n305|0|904K|0|n305|0|0|00:00:00|n305|0|00:00:00|1|1|00:00:25|COMPLETED|0:0|2.19M|0|0|0|18000Mn|0|0|n305|65534|0|0|n305|65534|0||||cpu=1,mem=18000M,node=1| 1672455_2|1672464|MV_mock-A47933_batch-uKEUyUuWbi2mgd75KjP4k5|short||||||||||||||||||1|00:00:27|COMPLETED|0:0||Unknown|Unknown|Unknown|18000Mn||||||||||||cpu=1,mem=18000M,node=1|cpu=1,mem=18000M,node=1| 1672455_2.batch|1672464.batch|batch||125588K|n305|0|125588K|900K|n305|0|900K|0|n305|0|0|00:00:00|n305|0|00:00:00|1|1|00:00:27|COMPLETED|0:0|2.19M|0|0|0|18000Mn|0|0|n305|65534|0|0|n305|65534|0||||cpu=1,mem=18000M,node=1| 1672465|1672465|MS_batch-uKEUyUuWbi2mgd75KjP4k5|short||||||||||||||||||1|00:00:02|COMPLETED|0:0||Unknown|Unknown|Unknown|16000Mn||||||||||||cpu=1,mem=16000M,node=1|cpu=1,mem=16000M,node=1| 1672465.batch|1672465.batch|batch||125588K|n305|0|125588K|896K|n305|0|896K|0|n305|0|0|00:00:00|n305|0|00:00:00|1|1|00:00:02|COMPLETED|0:0|2.19M|0|0|0|16000Mn|0|0|n305|65534|0|0|n305|65534|0||||cpu=1,mem=16000M,node=1| """.encode('utf8') patch_check.return_value = content job = _job.ArrayJob(output_dir='temp', job_ident='1672457', task_list=3, stage='validate') _scheduler.SlurmScheduler().update_info(job) self.assertEqual(_constants.JOB_STATUS.COMPLETED, job.status) self.assertEqual(3, len(job.task_list))
def test_single_job(self, patcher): sched = _scheduler.SlurmScheduler() job = _job.Job(SUBCOMMAND.VALIDATE, '', job_ident='1234') sched.cancel(job) self.assertEqual(_constants.JOB_STATUS.CANCELLED, job.status) patcher.assert_called_with(['scancel', '1234'])
def test_job_array(self): content = """ JobId=1673301 ArrayJobId=1673301 ArrayTaskId=3 JobName=subtest.sh UserId=creisle(1365) GroupId=users(100) MCS_label=N/A Priority=31 Nice=0 Account=all QOS=normal JobState=RUNNING Reason=None Dependency=(null) Requeue=1 Restarts=0 BatchFlag=1 Reboot=0 ExitCode=0:0 RunTime=00:00:11 TimeLimit=90-00:00:00 TimeMin=N/A SubmitTime=2018-05-24T11:38:28 EligibleTime=2018-05-24T11:38:28 StartTime=2018-05-24T11:38:29 EndTime=2018-08-22T11:38:29 Deadline=N/A PreemptTime=None SuspendTime=None SecsPreSuspend=0 Partition=all AllocNode:Sid=n104:47409 ReqNodeList=(null) ExcNodeList=(null) NodeList=n245 BatchHost=n245 NumNodes=1 NumCPUs=1 NumTasks=1 CPUs/Task=1 ReqB:S:C:T=0:0:*:* TRES=cpu=1,mem=7000M,node=1 Socks/Node=* NtasksPerN:B:S:C=0:0:*:* CoreSpec=* MinCPUsNode=1 MinMemoryCPU=7000M MinTmpDiskNode=0 Features=(null) DelayBoot=00:00:00 Gres=(null) Reservation=(null) OverSubscribe=OK Contiguous=0 Licenses=(null) Network=(null) Command=/projects/trans_scratch/validations/workspace/creisle/temp/subtest.sh WorkDir=/projects/trans_scratch/validations/workspace/creisle/temp StdErr=/projects/trans_scratch/validations/workspace/creisle/temp/slurm-1673301_3.out StdIn=/dev/null StdOut=/projects/trans_scratch/validations/workspace/creisle/temp/slurm-1673301_3.out Power= JobId=1673303 ArrayJobId=1673301 ArrayTaskId=2 JobName=subtest.sh UserId=creisle(1365) GroupId=users(100) MCS_label=N/A Priority=31 Nice=0 Account=all QOS=normal JobState=RUNNING Reason=None Dependency=(null) Requeue=1 Restarts=0 BatchFlag=1 Reboot=0 ExitCode=0:0 RunTime=00:00:11 TimeLimit=90-00:00:00 TimeMin=N/A SubmitTime=2018-05-24T11:38:28 EligibleTime=2018-05-24T11:38:28 StartTime=2018-05-24T11:38:29 EndTime=2018-08-22T11:38:29 Deadline=N/A PreemptTime=None SuspendTime=None SecsPreSuspend=0 Partition=all AllocNode:Sid=n104:47409 ReqNodeList=(null) ExcNodeList=(null) NodeList=n235 BatchHost=n235 NumNodes=1 NumCPUs=1 NumTasks=1 CPUs/Task=1 ReqB:S:C:T=0:0:*:* TRES=cpu=1,mem=7000M,node=1 Socks/Node=* NtasksPerN:B:S:C=0:0:*:* CoreSpec=* MinCPUsNode=1 MinMemoryCPU=7000M MinTmpDiskNode=0 Features=(null) DelayBoot=00:00:00 Gres=(null) Reservation=(null) OverSubscribe=OK Contiguous=0 Licenses=(null) Network=(null) Command=/projects/trans_scratch/validations/workspace/creisle/temp/subtest.sh WorkDir=/projects/trans_scratch/validations/workspace/creisle/temp StdErr=/projects/trans_scratch/validations/workspace/creisle/temp/slurm-1673301_2.out StdIn=/dev/null StdOut=/projects/trans_scratch/validations/workspace/creisle/temp/slurm-1673301_2.out Power= JobId=1673302 ArrayJobId=1673301 ArrayTaskId=1 JobName=subtest.sh UserId=creisle(1365) GroupId=users(100) MCS_label=N/A Priority=31 Nice=0 Account=all QOS=normal JobState=RUNNING Reason=None Dependency=(null) Requeue=1 Restarts=0 BatchFlag=1 Reboot=0 ExitCode=0:0 RunTime=00:00:11 TimeLimit=90-00:00:00 TimeMin=N/A SubmitTime=2018-05-24T11:38:28 EligibleTime=2018-05-24T11:38:28 StartTime=2018-05-24T11:38:29 EndTime=2018-08-22T11:38:29 Deadline=N/A PreemptTime=None SuspendTime=None SecsPreSuspend=0 Partition=all AllocNode:Sid=n104:47409 ReqNodeList=(null) ExcNodeList=(null) NodeList=n137 BatchHost=n137 NumNodes=1 NumCPUs=1 NumTasks=1 CPUs/Task=1 ReqB:S:C:T=0:0:*:* TRES=cpu=1,mem=7000M,node=1 Socks/Node=* NtasksPerN:B:S:C=0:0:*:* CoreSpec=* MinCPUsNode=1 MinMemoryCPU=7000M MinTmpDiskNode=0 Features=(null) DelayBoot=00:00:00 Gres=(null) Reservation=(null) OverSubscribe=OK Contiguous=0 Licenses=(null) Network=(null) Command=/projects/trans_scratch/validations/workspace/creisle/temp/subtest.sh WorkDir=/projects/trans_scratch/validations/workspace/creisle/temp StdErr=/projects/trans_scratch/validations/workspace/creisle/temp/slurm-1673301_1.out StdIn=/dev/null StdOut=/projects/trans_scratch/validations/workspace/creisle/temp/slurm-1673301_1.out Power= """ rows = _scheduler.SlurmScheduler().parse_scontrol_show(content) self.assertEqual(3, len(rows))