def test_bacct_killed(): """Test parsing `bacct -l` output for a canceled job.""" lsf = LsfLrms( name='test', architecture=gc3libs.Run.Arch.X86_64, max_cores=1, max_cores_per_job=1, max_memory_per_core=1 * GB, max_walltime=1 * hours, auth=None, # ignored if `transport` is `local` frontend='localhost', transport='local', bacct='bacct') acct = lsf._parse_acct_output(""" Accounting information about jobs that are: - submitted by all users. - accounted on all projects. - completed normally or exited - executed on all hosts. - submitted to all queues. - accounted on all service classes. ------------------------------------------------------------------------------ Job <3224113>, User <rmurri>, Project <default>, Status <EXIT>, Queue <pub.1h>, Command <sleep 300>, Share group charged </lsf_biol_all/ls f_aeber/rmurri> Fri Oct 5 17:49:35: Submitted from host <brutus4>, CWD <$HOME>, Output File <l sf.o%J>; Fri Oct 5 17:50:35: Dispatched to <a3191>; Fri Oct 5 17:51:30: Completed <exit>; TERM_OWNER: job killed by owner. Accounting information about this job: Share group charged </lsf_biol_all/lsf_aeber/rmurri> CPU_T WAIT TURNAROUND STATUS HOG_FACTOR MEM SWAP 0.04 60 115 exit 0.0003 1M 34M ------------------------------------------------------------------------------ SUMMARY: ( time unit: second ) Total number of done jobs: 0 Total number of exited jobs: 1 Total CPU time consumed: 0.0 Average CPU time consumed: 0.0 Maximum CPU time of a job: 0.0 Minimum CPU time of a job: 0.0 Total wait time in queues: 60.0 Average wait time in queue: 60.0 Maximum wait time in queue: 60.0 Minimum wait time in queue: 60.0 Average turnaround time: 115 (seconds/job) Maximum turnaround time: 115 Minimum turnaround time: 115 Average hog factor of a job: 0.00 ( cpu time / turnaround time ) Maximum hog factor of a job: 0.00 Minimum hog factor of a job: 0.00 """) assert_equal(acct['duration'], Duration('55s')) assert_equal(acct['used_cpu_time'], Duration('0.04s')) assert_equal(acct['max_used_memory'], Memory('35MB')) # timestamps year = datetime.date.today().year assert_equal(acct['lsf_submission_time'], datetime.datetime(year, 10, 5, 17, 49, 35)) assert_equal(acct['lsf_start_time'], datetime.datetime(year, 10, 5, 17, 50, 35)) assert_equal(acct['lsf_completion_time'], datetime.datetime(year, 10, 5, 17, 51, 30))
def test_bacct_done1(): """Test parsing `bacct -l` output for a not-so-trivial job.""" lsf = LsfLrms( name='test', architecture=gc3libs.Run.Arch.X86_64, max_cores=1, max_cores_per_job=1, max_memory_per_core=1 * GB, max_walltime=1 * hours, auth=None, # ignored if `transport` is `local` frontend='localhost', transport='local', bacct='bacct') acct = lsf._parse_acct_output(""" Accounting information about jobs that are: - submitted by all users. - accounted on all projects. - completed normally or exited - executed on all hosts. - submitted to all queues. - accounted on all service classes. ------------------------------------------------------------------------------ Job <3329618>, User <rmurri>, Project <default>, Status <DONE>, Queue <pub.1h>, Command <md5sum lsf.o3224113 lsf.o3224132>, Share group ch arged </lsf_biol_all/lsf_aeber/rmurri> Mon Oct 8 17:08:54: Submitted from host <brutus4>, CWD <$HOME>, Output File <l sf.o%J>; Mon Oct 8 17:10:01: Dispatched to <a3041>; Mon Oct 8 17:10:07: Completed <done>. Accounting information about this job: Share group charged </lsf_biol_all/lsf_aeber/rmurri> CPU_T WAIT TURNAROUND STATUS HOG_FACTOR MEM SWAP 0.04 67 73 done 0.0005 3M 34M ------------------------------------------------------------------------------ SUMMARY: ( time unit: second ) Total number of done jobs: 1 Total number of exited jobs: 0 Total CPU time consumed: 0.0 Average CPU time consumed: 0.0 Maximum CPU time of a job: 0.0 Minimum CPU time of a job: 0.0 Total wait time in queues: 67.0 Average wait time in queue: 67.0 Maximum wait time in queue: 67.0 Minimum wait time in queue: 67.0 Average turnaround time: 73 (seconds/job) Maximum turnaround time: 73 Minimum turnaround time: 73 Average hog factor of a job: 0.00 ( cpu time / turnaround time ) Maximum hog factor of a job: 0.00 Minimum hog factor of a job: 0.00 """) assert_equal(acct['duration'], Duration('6s')) assert_equal(acct['used_cpu_time'], Duration('0.04s')) assert_equal(acct['max_used_memory'], Memory('37MB')) # timestamps year = datetime.date.today().year assert_equal(acct['lsf_submission_time'], datetime.datetime(year, 10, 8, 17, 8, 54)) assert_equal(acct['lsf_start_time'], datetime.datetime(year, 10, 8, 17, 10, 1)) assert_equal(acct['lsf_completion_time'], datetime.datetime(year, 10, 8, 17, 10, 7))
def test_bacct_killed(): """Test parsing `bacct -l` output for a canceled job.""" lsf = LsfLrms(name='test', architecture=gc3libs.Run.Arch.X86_64, max_cores=1, max_cores_per_job=1, max_memory_per_core=1 * GB, max_walltime=1 * hours, auth=None, # ignored if `transport` is `local` frontend='localhost', transport='local', bacct='bacct') acct = lsf._parse_acct_output(""" Accounting information about jobs that are: - submitted by all users. - accounted on all projects. - completed normally or exited - executed on all hosts. - submitted to all queues. - accounted on all service classes. ------------------------------------------------------------------------------ Job <3224113>, User <rmurri>, Project <default>, Status <EXIT>, Queue <pub.1h>, Command <sleep 300>, Share group charged </lsf_biol_all/ls f_aeber/rmurri> Fri Oct 5 17:49:35: Submitted from host <brutus4>, CWD <$HOME>, Output File <l sf.o%J>; Fri Oct 5 17:50:35: Dispatched to <a3191>; Fri Oct 5 17:51:30: Completed <exit>; TERM_OWNER: job killed by owner. Accounting information about this job: Share group charged </lsf_biol_all/lsf_aeber/rmurri> CPU_T WAIT TURNAROUND STATUS HOG_FACTOR MEM SWAP 0.04 60 115 exit 0.0003 1M 34M ------------------------------------------------------------------------------ SUMMARY: ( time unit: second ) Total number of done jobs: 0 Total number of exited jobs: 1 Total CPU time consumed: 0.0 Average CPU time consumed: 0.0 Maximum CPU time of a job: 0.0 Minimum CPU time of a job: 0.0 Total wait time in queues: 60.0 Average wait time in queue: 60.0 Maximum wait time in queue: 60.0 Minimum wait time in queue: 60.0 Average turnaround time: 115 (seconds/job) Maximum turnaround time: 115 Minimum turnaround time: 115 Average hog factor of a job: 0.00 ( cpu time / turnaround time ) Maximum hog factor of a job: 0.00 Minimum hog factor of a job: 0.00 """) assert_equal(acct['duration'], Duration('55s')) assert_equal(acct['used_cpu_time'], Duration('0.04s')) assert_equal(acct['max_used_memory'], Memory('35MB')) # timestamps year = datetime.date.today().year assert_equal(acct['lsf_submission_time'], datetime.datetime(year, 10, 5, 17, 49, 35)) assert_equal(acct['lsf_start_time'], datetime.datetime(year, 10, 5, 17, 50, 35)) assert_equal(acct['lsf_completion_time'], datetime.datetime(year, 10, 5, 17, 51, 30))
def test_bacct_done1(): """Test parsing `bacct -l` output for a not-so-trivial job.""" lsf = LsfLrms(name='test', architecture=gc3libs.Run.Arch.X86_64, max_cores=1, max_cores_per_job=1, max_memory_per_core=1 * GB, max_walltime=1 * hours, auth=None, # ignored if `transport` is `local` frontend='localhost', transport='local', bacct='bacct') acct = lsf._parse_acct_output(""" Accounting information about jobs that are: - submitted by all users. - accounted on all projects. - completed normally or exited - executed on all hosts. - submitted to all queues. - accounted on all service classes. ------------------------------------------------------------------------------ Job <3329618>, User <rmurri>, Project <default>, Status <DONE>, Queue <pub.1h>, Command <md5sum lsf.o3224113 lsf.o3224132>, Share group ch arged </lsf_biol_all/lsf_aeber/rmurri> Mon Oct 8 17:08:54: Submitted from host <brutus4>, CWD <$HOME>, Output File <l sf.o%J>; Mon Oct 8 17:10:01: Dispatched to <a3041>; Mon Oct 8 17:10:07: Completed <done>. Accounting information about this job: Share group charged </lsf_biol_all/lsf_aeber/rmurri> CPU_T WAIT TURNAROUND STATUS HOG_FACTOR MEM SWAP 0.04 67 73 done 0.0005 3M 34M ------------------------------------------------------------------------------ SUMMARY: ( time unit: second ) Total number of done jobs: 1 Total number of exited jobs: 0 Total CPU time consumed: 0.0 Average CPU time consumed: 0.0 Maximum CPU time of a job: 0.0 Minimum CPU time of a job: 0.0 Total wait time in queues: 67.0 Average wait time in queue: 67.0 Maximum wait time in queue: 67.0 Minimum wait time in queue: 67.0 Average turnaround time: 73 (seconds/job) Maximum turnaround time: 73 Minimum turnaround time: 73 Average hog factor of a job: 0.00 ( cpu time / turnaround time ) Maximum hog factor of a job: 0.00 Minimum hog factor of a job: 0.00 """) assert_equal(acct['duration'], Duration('6s')) assert_equal(acct['used_cpu_time'], Duration('0.04s')) assert_equal(acct['max_used_memory'], Memory('37MB')) # timestamps year = datetime.date.today().year assert_equal(acct['lsf_submission_time'], datetime.datetime(year, 10, 8, 17, 8, 54)) assert_equal(acct['lsf_start_time'], datetime.datetime(year, 10, 8, 17, 10, 1)) assert_equal(acct['lsf_completion_time'], datetime.datetime(year, 10, 8, 17, 10, 7))
def test_bjobs_output_for_accounting(): lsf = LsfLrms( name='test', architecture=gc3libs.Run.Arch.X86_64, max_cores=1, max_cores_per_job=1, max_memory_per_core=1 * GB, max_walltime=1 * hours, auth=None, # ignored if `transport` is `local` frontend='localhost', transport='local') bjobs_output = """ Job <131851>, Job Name <ChromaExtractShort>, User <wwolski>, Project <default>, Status <DONE>, Queue <pub.8h>, Job Priority <50>, Command <ChromatogramExtractor -in /cluster/scratch/malars/openswa th/data/AQUA_fixed_water/split_napedro_L120224_001_SW-400A QUA_no_background_2ul_dilution_10/split_napedro_L120224_00 1_SW-400AQUA_no_background_2ul_dilution_10_28.mzML.gz -tr /cluster/scratch/malars/openswath/assays/iRT/DIA_iRT.TraML -out split_napedro_L120224_001_SW-400AQUA_no_background_2 ul_dilution_10_28._rtnorm.chrom.mzML -is_swath -min_upper_ edge_dist 1 -threads 2>, Share group charged </lsf_biol_al l/lsf_biol_other/wwolski> Tue Jul 24 10:03:15: Submitted from host <brutus3>, CWD <$HOME/.gc3pie_jobs/lrm s_job.YNZmU17755/.>, Output File <lsf.o%J>, Requested Reso urces <select[mem<70000 && lustre] order[-ut] rusage[mem=1 000,m=1]>, Login Shell </bin/sh>, Specified Hosts <thin+9> , <single+8>, <smp16+6>, <smp24+5>, <smp48+4>; RUNLIMIT 480.0 min of a6122 Tue Jul 24 10:04:19: Started on <a6122>, Execution Home </cluster/home/biol/wwo lski>, Execution CWD </cluster/home/biol/wwolski/.gc3pie_j obs/lrms_job.YNZmU17755/.>; Tue Jul 24 10:05:45: Done successfully. The CPU time used is 2.1 seconds. MEMORY USAGE: MAX MEM: 41 Mbytes; AVG MEM: 41 Mbytes SCHEDULING PARAMETERS: r15s r1m r15m ut pg io ls it tmp swp mem loadSched - - - - - - - - 1000M - - loadStop - - - - - - - - - - - scratch xs s m l xl sp loadSched 4000.0 - - - - - - loadStop - - - - - - - """ # Also parse the output of jobs to get accounting information acct = lsf._parse_acct_output(bjobs_output) assert_equal(acct['duration'], Duration('86s')) assert_equal(acct['used_cpu_time'], Duration('2.1s')) assert_equal(acct['max_used_memory'], Memory('41MB'))
def test_bjobs_output_for_accounting(): lsf = LsfLrms(name='test', architecture=gc3libs.Run.Arch.X86_64, max_cores=1, max_cores_per_job=1, max_memory_per_core=1 * GB, max_walltime=1 * hours, auth=None, # ignored if `transport` is `local` frontend='localhost', transport='local') bjobs_output = """ Job <131851>, Job Name <ChromaExtractShort>, User <wwolski>, Project <default>, Status <DONE>, Queue <pub.8h>, Job Priority <50>, Command <ChromatogramExtractor -in /cluster/scratch/malars/openswa th/data/AQUA_fixed_water/split_napedro_L120224_001_SW-400A QUA_no_background_2ul_dilution_10/split_napedro_L120224_00 1_SW-400AQUA_no_background_2ul_dilution_10_28.mzML.gz -tr /cluster/scratch/malars/openswath/assays/iRT/DIA_iRT.TraML -out split_napedro_L120224_001_SW-400AQUA_no_background_2 ul_dilution_10_28._rtnorm.chrom.mzML -is_swath -min_upper_ edge_dist 1 -threads 2>, Share group charged </lsf_biol_al l/lsf_biol_other/wwolski> Tue Jul 24 10:03:15: Submitted from host <brutus3>, CWD <$HOME/.gc3pie_jobs/lrm s_job.YNZmU17755/.>, Output File <lsf.o%J>, Requested Reso urces <select[mem<70000 && lustre] order[-ut] rusage[mem=1 000,m=1]>, Login Shell </bin/sh>, Specified Hosts <thin+9> , <single+8>, <smp16+6>, <smp24+5>, <smp48+4>; RUNLIMIT 480.0 min of a6122 Tue Jul 24 10:04:19: Started on <a6122>, Execution Home </cluster/home/biol/wwo lski>, Execution CWD </cluster/home/biol/wwolski/.gc3pie_j obs/lrms_job.YNZmU17755/.>; Tue Jul 24 10:05:45: Done successfully. The CPU time used is 2.1 seconds. MEMORY USAGE: MAX MEM: 41 Mbytes; AVG MEM: 41 Mbytes SCHEDULING PARAMETERS: r15s r1m r15m ut pg io ls it tmp swp mem loadSched - - - - - - - - 1000M - - loadStop - - - - - - - - - - - scratch xs s m l xl sp loadSched 4000.0 - - - - - - loadStop - - - - - - - """ # Also parse the output of jobs to get accounting information acct = lsf._parse_acct_output(bjobs_output, '') assert_equal(acct['duration'], Duration('86s')) assert_equal(acct['used_cpu_time'], Duration('2.1s')) assert_equal(acct['max_used_memory'], Memory('41MB'))
def test_bacct_done0(): """Test parsing accounting information of a <sleep 300> job.""" # gotten with `bacct -l "jobid"` acct = LsfLrms._parse_acct_output(""" Accounting information about jobs that are: - submitted by all users. - accounted on all projects. - completed normally or exited - executed on all hosts. - submitted to all queues. - accounted on all service classes. ------------------------------------------------------------------------------ Job <3329613>, User <rmurri>, Project <default>, Status <DONE>, Queue <pub.1h>, Command <sleep 60>, Share group charged </lsf_biol_all/lsf _aeber/rmurri> Mon Oct 8 17:07:54: Submitted from host <brutus4>, CWD <$HOME>, Output File <l sf.o%J>; Mon Oct 8 17:08:44: Dispatched to <a3201>; Mon Oct 8 17:09:51: Completed <done>. Accounting information about this job: Share group charged </lsf_biol_all/lsf_aeber/rmurri> CPU_T WAIT TURNAROUND STATUS HOG_FACTOR MEM SWAP 0.08 50 117 done 0.0007 5M 222M ------------------------------------------------------------------------------ SUMMARY: ( time unit: second ) Total number of done jobs: 1 Total number of exited jobs: 0 Total CPU time consumed: 0.1 Average CPU time consumed: 0.1 Maximum CPU time of a job: 0.1 Minimum CPU time of a job: 0.1 Total wait time in queues: 50.0 Average wait time in queue: 50.0 Maximum wait time in queue: 50.0 Minimum wait time in queue: 50.0 Average turnaround time: 117 (seconds/job) Maximum turnaround time: 117 Minimum turnaround time: 117 Average hog factor of a job: 0.00 ( cpu time / turnaround time ) Maximum hog factor of a job: 0.00 Minimum hog factor of a job: 0.00 """) assert_equal(acct['duration'], Duration('67s')) assert_equal(acct['used_cpu_time'], Duration('0.08s')) assert_equal(acct['max_used_memory'], Memory('227MB')) # timestamps year = datetime.date.today().year assert_equal(acct['lsf_submission_time'], datetime.datetime(year, 10, 8, 17, 7, 54)) assert_equal(acct['lsf_start_time'], datetime.datetime(year, 10, 8, 17, 8, 44)) assert_equal(acct['lsf_completion_time'], datetime.datetime(year, 10, 8, 17, 9, 51))
def test_bacct_done0(): """Test parsing accounting information of a <sleep 300> job.""" # gotten with `bacct -l "jobid"` lsf = LsfLrms(name='test', architecture=gc3libs.Run.Arch.X86_64, max_cores=1, max_cores_per_job=1, max_memory_per_core=1 * GB, max_walltime=1 * hours, auth=None, # ignored if `transport` is `local` frontend='localhost', transport='local', bacct='bacct') acct = lsf._parse_acct_output(""" Accounting information about jobs that are: - submitted by all users. - accounted on all projects. - completed normally or exited - executed on all hosts. - submitted to all queues. - accounted on all service classes. ------------------------------------------------------------------------------ Job <3329613>, User <rmurri>, Project <default>, Status <DONE>, Queue <pub.1h>, Command <sleep 60>, Share group charged </lsf_biol_all/lsf _aeber/rmurri> Mon Oct 8 17:07:54: Submitted from host <brutus4>, CWD <$HOME>, Output File <l sf.o%J>; Mon Oct 8 17:08:44: Dispatched to <a3201>; Mon Oct 8 17:09:51: Completed <done>. Accounting information about this job: Share group charged </lsf_biol_all/lsf_aeber/rmurri> CPU_T WAIT TURNAROUND STATUS HOG_FACTOR MEM SWAP 0.08 50 117 done 0.0007 5M 222M ------------------------------------------------------------------------------ SUMMARY: ( time unit: second ) Total number of done jobs: 1 Total number of exited jobs: 0 Total CPU time consumed: 0.1 Average CPU time consumed: 0.1 Maximum CPU time of a job: 0.1 Minimum CPU time of a job: 0.1 Total wait time in queues: 50.0 Average wait time in queue: 50.0 Maximum wait time in queue: 50.0 Minimum wait time in queue: 50.0 Average turnaround time: 117 (seconds/job) Maximum turnaround time: 117 Minimum turnaround time: 117 Average hog factor of a job: 0.00 ( cpu time / turnaround time ) Maximum hog factor of a job: 0.00 Minimum hog factor of a job: 0.00 """, # STDERR '') assert acct['duration'] == Duration('67s') assert acct['used_cpu_time'] == Duration('0.08s') assert acct['max_used_memory'] == Memory('227MB') # timestamps year = datetime.date.today().year assert (acct['lsf_submission_time'] == datetime.datetime(year, 10, 8, 17, 7, 54)) assert (acct['lsf_start_time'] == datetime.datetime(year, 10, 8, 17, 8, 44)) assert (acct['lsf_completion_time'] == datetime.datetime(year, 10, 8, 17, 9, 51))