def test_bacct_killed():
"""Test parsing `bacct -l` output for a canceled job."""
lsf = LsfLrms(
name='test',
architecture=gc3libs.Run.Arch.X86_64,
max_cores=1,
max_cores_per_job=1,
max_memory_per_core=1 * GB,
max_walltime=1 * hours,
auth=None, # ignored if `transport` is `local`
frontend='localhost',
transport='local',
bacct='bacct')
acct = lsf._parse_acct_output("""
Accounting information about jobs that are:
- submitted by all users.
- accounted on all projects.
- completed normally or exited
- executed on all hosts.
- submitted to all queues.
- accounted on all service classes.
------------------------------------------------------------------------------
Job <3224113>, User <rmurri>, Project <default>, Status <EXIT>, Queue <pub.1h>,
Command <sleep 300>, Share group charged </lsf_biol_all/ls
f_aeber/rmurri>
Fri Oct 5 17:49:35: Submitted from host <brutus4>, CWD <$HOME>, Output File <l
sf.o%J>;
Fri Oct 5 17:50:35: Dispatched to <a3191>;
Fri Oct 5 17:51:30: Completed <exit>; TERM_OWNER: job killed by owner.
Accounting information about this job:
Share group charged </lsf_biol_all/lsf_aeber/rmurri>
CPU_T WAIT TURNAROUND STATUS HOG_FACTOR MEM SWAP
0.04 60 115 exit 0.0003 1M 34M
------------------------------------------------------------------------------
SUMMARY: ( time unit: second )
Total number of done jobs: 0 Total number of exited jobs: 1
Total CPU time consumed: 0.0 Average CPU time consumed: 0.0
Maximum CPU time of a job: 0.0 Minimum CPU time of a job: 0.0
Total wait time in queues: 60.0
Average wait time in queue: 60.0
Maximum wait time in queue: 60.0 Minimum wait time in queue: 60.0
Average turnaround time: 115 (seconds/job)
Maximum turnaround time: 115 Minimum turnaround time: 115
Average hog factor of a job: 0.00 ( cpu time / turnaround time )
Maximum hog factor of a job: 0.00 Minimum hog factor of a job: 0.00
""")
assert_equal(acct['duration'], Duration('55s'))
assert_equal(acct['used_cpu_time'], Duration('0.04s'))
assert_equal(acct['max_used_memory'], Memory('35MB'))
# timestamps
year = datetime.date.today().year
assert_equal(acct['lsf_submission_time'],
datetime.datetime(year, 10, 5, 17, 49, 35))
assert_equal(acct['lsf_start_time'],
datetime.datetime(year, 10, 5, 17, 50, 35))
assert_equal(acct['lsf_completion_time'],
datetime.datetime(year, 10, 5, 17, 51, 30))
def test_bacct_done1():
"""Test parsing `bacct -l` output for a not-so-trivial job."""
lsf = LsfLrms(
name='test',
architecture=gc3libs.Run.Arch.X86_64,
max_cores=1,
max_cores_per_job=1,
max_memory_per_core=1 * GB,
max_walltime=1 * hours,
auth=None, # ignored if `transport` is `local`
frontend='localhost',
transport='local',
bacct='bacct')
acct = lsf._parse_acct_output("""
Accounting information about jobs that are:
- submitted by all users.
- accounted on all projects.
- completed normally or exited
- executed on all hosts.
- submitted to all queues.
- accounted on all service classes.
------------------------------------------------------------------------------
Job <3329618>, User <rmurri>, Project <default>, Status <DONE>, Queue <pub.1h>,
Command <md5sum lsf.o3224113 lsf.o3224132>, Share group ch
arged </lsf_biol_all/lsf_aeber/rmurri>
Mon Oct 8 17:08:54: Submitted from host <brutus4>, CWD <$HOME>, Output File <l
sf.o%J>;
Mon Oct 8 17:10:01: Dispatched to <a3041>;
Mon Oct 8 17:10:07: Completed <done>.
Accounting information about this job:
Share group charged </lsf_biol_all/lsf_aeber/rmurri>
CPU_T WAIT TURNAROUND STATUS HOG_FACTOR MEM SWAP
0.04 67 73 done 0.0005 3M 34M
------------------------------------------------------------------------------
SUMMARY: ( time unit: second )
Total number of done jobs: 1 Total number of exited jobs: 0
Total CPU time consumed: 0.0 Average CPU time consumed: 0.0
Maximum CPU time of a job: 0.0 Minimum CPU time of a job: 0.0
Total wait time in queues: 67.0
Average wait time in queue: 67.0
Maximum wait time in queue: 67.0 Minimum wait time in queue: 67.0
Average turnaround time: 73 (seconds/job)
Maximum turnaround time: 73 Minimum turnaround time: 73
Average hog factor of a job: 0.00 ( cpu time / turnaround time )
Maximum hog factor of a job: 0.00 Minimum hog factor of a job: 0.00
""")
assert_equal(acct['duration'], Duration('6s'))
assert_equal(acct['used_cpu_time'], Duration('0.04s'))
assert_equal(acct['max_used_memory'], Memory('37MB'))
# timestamps
year = datetime.date.today().year
assert_equal(acct['lsf_submission_time'],
datetime.datetime(year, 10, 8, 17, 8, 54))
assert_equal(acct['lsf_start_time'],
datetime.datetime(year, 10, 8, 17, 10, 1))
assert_equal(acct['lsf_completion_time'],
datetime.datetime(year, 10, 8, 17, 10, 7))
def test_bacct_killed():
"""Test parsing `bacct -l` output for a canceled job."""
lsf = LsfLrms(name='test',
architecture=gc3libs.Run.Arch.X86_64,
max_cores=1,
max_cores_per_job=1,
max_memory_per_core=1 * GB,
max_walltime=1 * hours,
auth=None, # ignored if `transport` is `local`
frontend='localhost',
transport='local',
bacct='bacct')
acct = lsf._parse_acct_output("""
Accounting information about jobs that are:
- submitted by all users.
- accounted on all projects.
- completed normally or exited
- executed on all hosts.
- submitted to all queues.
- accounted on all service classes.
------------------------------------------------------------------------------
Job <3224113>, User <rmurri>, Project <default>, Status <EXIT>, Queue <pub.1h>,
Command <sleep 300>, Share group charged </lsf_biol_all/ls
f_aeber/rmurri>
Fri Oct 5 17:49:35: Submitted from host <brutus4>, CWD <$HOME>, Output File <l
sf.o%J>;
Fri Oct 5 17:50:35: Dispatched to <a3191>;
Fri Oct 5 17:51:30: Completed <exit>; TERM_OWNER: job killed by owner.
Accounting information about this job:
Share group charged </lsf_biol_all/lsf_aeber/rmurri>
CPU_T WAIT TURNAROUND STATUS HOG_FACTOR MEM SWAP
0.04 60 115 exit 0.0003 1M 34M
------------------------------------------------------------------------------
SUMMARY: ( time unit: second )
Total number of done jobs: 0 Total number of exited jobs: 1
Total CPU time consumed: 0.0 Average CPU time consumed: 0.0
Maximum CPU time of a job: 0.0 Minimum CPU time of a job: 0.0
Total wait time in queues: 60.0
Average wait time in queue: 60.0
Maximum wait time in queue: 60.0 Minimum wait time in queue: 60.0
Average turnaround time: 115 (seconds/job)
Maximum turnaround time: 115 Minimum turnaround time: 115
Average hog factor of a job: 0.00 ( cpu time / turnaround time )
Maximum hog factor of a job: 0.00 Minimum hog factor of a job: 0.00
""")
assert_equal(acct['duration'], Duration('55s'))
assert_equal(acct['used_cpu_time'], Duration('0.04s'))
assert_equal(acct['max_used_memory'], Memory('35MB'))
# timestamps
year = datetime.date.today().year
assert_equal(acct['lsf_submission_time'],
datetime.datetime(year, 10, 5, 17, 49, 35))
assert_equal(acct['lsf_start_time'],
datetime.datetime(year, 10, 5, 17, 50, 35))
assert_equal(acct['lsf_completion_time'],
datetime.datetime(year, 10, 5, 17, 51, 30))
def test_bacct_done1():
"""Test parsing `bacct -l` output for a not-so-trivial job."""
lsf = LsfLrms(name='test',
architecture=gc3libs.Run.Arch.X86_64,
max_cores=1,
max_cores_per_job=1,
max_memory_per_core=1 * GB,
max_walltime=1 * hours,
auth=None, # ignored if `transport` is `local`
frontend='localhost',
transport='local',
bacct='bacct')
acct = lsf._parse_acct_output("""
Accounting information about jobs that are:
- submitted by all users.
- accounted on all projects.
- completed normally or exited
- executed on all hosts.
- submitted to all queues.
- accounted on all service classes.
------------------------------------------------------------------------------
Job <3329618>, User <rmurri>, Project <default>, Status <DONE>, Queue <pub.1h>,
Command <md5sum lsf.o3224113 lsf.o3224132>, Share group ch
arged </lsf_biol_all/lsf_aeber/rmurri>
Mon Oct 8 17:08:54: Submitted from host <brutus4>, CWD <$HOME>, Output File <l
sf.o%J>;
Mon Oct 8 17:10:01: Dispatched to <a3041>;
Mon Oct 8 17:10:07: Completed <done>.
Accounting information about this job:
Share group charged </lsf_biol_all/lsf_aeber/rmurri>
CPU_T WAIT TURNAROUND STATUS HOG_FACTOR MEM SWAP
0.04 67 73 done 0.0005 3M 34M
------------------------------------------------------------------------------
SUMMARY: ( time unit: second )
Total number of done jobs: 1 Total number of exited jobs: 0
Total CPU time consumed: 0.0 Average CPU time consumed: 0.0
Maximum CPU time of a job: 0.0 Minimum CPU time of a job: 0.0
Total wait time in queues: 67.0
Average wait time in queue: 67.0
Maximum wait time in queue: 67.0 Minimum wait time in queue: 67.0
Average turnaround time: 73 (seconds/job)
Maximum turnaround time: 73 Minimum turnaround time: 73
Average hog factor of a job: 0.00 ( cpu time / turnaround time )
Maximum hog factor of a job: 0.00 Minimum hog factor of a job: 0.00
""")
assert_equal(acct['duration'], Duration('6s'))
assert_equal(acct['used_cpu_time'], Duration('0.04s'))
assert_equal(acct['max_used_memory'], Memory('37MB'))
# timestamps
year = datetime.date.today().year
assert_equal(acct['lsf_submission_time'],
datetime.datetime(year, 10, 8, 17, 8, 54))
assert_equal(acct['lsf_start_time'],
datetime.datetime(year, 10, 8, 17, 10, 1))
assert_equal(acct['lsf_completion_time'],
datetime.datetime(year, 10, 8, 17, 10, 7))
def test_bjobs_output_for_accounting():
lsf = LsfLrms(
name='test',
architecture=gc3libs.Run.Arch.X86_64,
max_cores=1,
max_cores_per_job=1,
max_memory_per_core=1 * GB,
max_walltime=1 * hours,
auth=None, # ignored if `transport` is `local`
frontend='localhost',
transport='local')
bjobs_output = """
Job <131851>, Job Name <ChromaExtractShort>, User <wwolski>, Project <default>,
Status <DONE>, Queue <pub.8h>, Job Priority <50>, Command
<ChromatogramExtractor -in /cluster/scratch/malars/openswa
th/data/AQUA_fixed_water/split_napedro_L120224_001_SW-400A
QUA_no_background_2ul_dilution_10/split_napedro_L120224_00
1_SW-400AQUA_no_background_2ul_dilution_10_28.mzML.gz -tr
/cluster/scratch/malars/openswath/assays/iRT/DIA_iRT.TraML
-out split_napedro_L120224_001_SW-400AQUA_no_background_2
ul_dilution_10_28._rtnorm.chrom.mzML -is_swath -min_upper_
edge_dist 1 -threads 2>, Share group charged </lsf_biol_al
l/lsf_biol_other/wwolski>
Tue Jul 24 10:03:15: Submitted from host <brutus3>, CWD <$HOME/.gc3pie_jobs/lrm
s_job.YNZmU17755/.>, Output File <lsf.o%J>, Requested Reso
urces <select[mem<70000 && lustre] order[-ut] rusage[mem=1
000,m=1]>, Login Shell </bin/sh>, Specified Hosts <thin+9>
, <single+8>, <smp16+6>, <smp24+5>, <smp48+4>;
RUNLIMIT
480.0 min of a6122
Tue Jul 24 10:04:19: Started on <a6122>, Execution Home </cluster/home/biol/wwo
lski>, Execution CWD </cluster/home/biol/wwolski/.gc3pie_j
obs/lrms_job.YNZmU17755/.>;
Tue Jul 24 10:05:45: Done successfully. The CPU time used is 2.1 seconds.
MEMORY USAGE:
MAX MEM: 41 Mbytes; AVG MEM: 41 Mbytes
SCHEDULING PARAMETERS:
r15s r1m r15m ut pg io ls it tmp swp mem
loadSched - - - - - - - - 1000M - -
loadStop - - - - - - - - - - -
scratch xs s m l xl sp
loadSched 4000.0 - - - - - -
loadStop - - - - - - -
"""
# Also parse the output of jobs to get accounting information
acct = lsf._parse_acct_output(bjobs_output)
assert_equal(acct['duration'], Duration('86s'))
assert_equal(acct['used_cpu_time'], Duration('2.1s'))
assert_equal(acct['max_used_memory'], Memory('41MB'))
def test_bjobs_output_for_accounting():
lsf = LsfLrms(name='test',
architecture=gc3libs.Run.Arch.X86_64,
max_cores=1,
max_cores_per_job=1,
max_memory_per_core=1 * GB,
max_walltime=1 * hours,
auth=None, # ignored if `transport` is `local`
frontend='localhost',
transport='local')
bjobs_output = """
Job <131851>, Job Name <ChromaExtractShort>, User <wwolski>, Project <default>,
Status <DONE>, Queue <pub.8h>, Job Priority <50>, Command
<ChromatogramExtractor -in /cluster/scratch/malars/openswa
th/data/AQUA_fixed_water/split_napedro_L120224_001_SW-400A
QUA_no_background_2ul_dilution_10/split_napedro_L120224_00
1_SW-400AQUA_no_background_2ul_dilution_10_28.mzML.gz -tr
/cluster/scratch/malars/openswath/assays/iRT/DIA_iRT.TraML
-out split_napedro_L120224_001_SW-400AQUA_no_background_2
ul_dilution_10_28._rtnorm.chrom.mzML -is_swath -min_upper_
edge_dist 1 -threads 2>, Share group charged </lsf_biol_al
l/lsf_biol_other/wwolski>
Tue Jul 24 10:03:15: Submitted from host <brutus3>, CWD <$HOME/.gc3pie_jobs/lrm
s_job.YNZmU17755/.>, Output File <lsf.o%J>, Requested Reso
urces <select[mem<70000 && lustre] order[-ut] rusage[mem=1
000,m=1]>, Login Shell </bin/sh>, Specified Hosts <thin+9>
, <single+8>, <smp16+6>, <smp24+5>, <smp48+4>;
RUNLIMIT
480.0 min of a6122
Tue Jul 24 10:04:19: Started on <a6122>, Execution Home </cluster/home/biol/wwo
lski>, Execution CWD </cluster/home/biol/wwolski/.gc3pie_j
obs/lrms_job.YNZmU17755/.>;
Tue Jul 24 10:05:45: Done successfully. The CPU time used is 2.1 seconds.
MEMORY USAGE:
MAX MEM: 41 Mbytes; AVG MEM: 41 Mbytes
SCHEDULING PARAMETERS:
r15s r1m r15m ut pg io ls it tmp swp mem
loadSched - - - - - - - - 1000M - -
loadStop - - - - - - - - - - -
scratch xs s m l xl sp
loadSched 4000.0 - - - - - -
loadStop - - - - - - -
"""
# Also parse the output of jobs to get accounting information
acct = lsf._parse_acct_output(bjobs_output, '')
assert_equal(acct['duration'], Duration('86s'))
assert_equal(acct['used_cpu_time'], Duration('2.1s'))
assert_equal(acct['max_used_memory'], Memory('41MB'))
def test_bacct_done0():
"""Test parsing accounting information of a <sleep 300> job."""
# gotten with `bacct -l "jobid"`
acct = LsfLrms._parse_acct_output("""
Accounting information about jobs that are:
- submitted by all users.
- accounted on all projects.
- completed normally or exited
- executed on all hosts.
- submitted to all queues.
- accounted on all service classes.
------------------------------------------------------------------------------
Job <3329613>, User <rmurri>, Project <default>, Status <DONE>, Queue <pub.1h>,
Command <sleep 60>, Share group charged </lsf_biol_all/lsf
_aeber/rmurri>
Mon Oct 8 17:07:54: Submitted from host <brutus4>, CWD <$HOME>, Output File <l
sf.o%J>;
Mon Oct 8 17:08:44: Dispatched to <a3201>;
Mon Oct 8 17:09:51: Completed <done>.
Accounting information about this job:
Share group charged </lsf_biol_all/lsf_aeber/rmurri>
CPU_T WAIT TURNAROUND STATUS HOG_FACTOR MEM SWAP
0.08 50 117 done 0.0007 5M 222M
------------------------------------------------------------------------------
SUMMARY: ( time unit: second )
Total number of done jobs: 1 Total number of exited jobs: 0
Total CPU time consumed: 0.1 Average CPU time consumed: 0.1
Maximum CPU time of a job: 0.1 Minimum CPU time of a job: 0.1
Total wait time in queues: 50.0
Average wait time in queue: 50.0
Maximum wait time in queue: 50.0 Minimum wait time in queue: 50.0
Average turnaround time: 117 (seconds/job)
Maximum turnaround time: 117 Minimum turnaround time: 117
Average hog factor of a job: 0.00 ( cpu time / turnaround time )
Maximum hog factor of a job: 0.00 Minimum hog factor of a job: 0.00
""")
assert_equal(acct['duration'], Duration('67s'))
assert_equal(acct['used_cpu_time'], Duration('0.08s'))
assert_equal(acct['max_used_memory'], Memory('227MB'))
# timestamps
year = datetime.date.today().year
assert_equal(acct['lsf_submission_time'], datetime.datetime(year, 10, 8, 17, 7, 54))
assert_equal(acct['lsf_start_time'], datetime.datetime(year, 10, 8, 17, 8, 44))
assert_equal(acct['lsf_completion_time'], datetime.datetime(year, 10, 8, 17, 9, 51))
def test_bacct_done0():
"""Test parsing accounting information of a <sleep 300> job."""
# gotten with `bacct -l "jobid"`
lsf = LsfLrms(name='test',
architecture=gc3libs.Run.Arch.X86_64,
max_cores=1,
max_cores_per_job=1,
max_memory_per_core=1 * GB,
max_walltime=1 * hours,
auth=None, # ignored if `transport` is `local`
frontend='localhost',
transport='local',
bacct='bacct')
acct = lsf._parse_acct_output("""
Accounting information about jobs that are:
- submitted by all users.
- accounted on all projects.
- completed normally or exited
- executed on all hosts.
- submitted to all queues.
- accounted on all service classes.
------------------------------------------------------------------------------
Job <3329613>, User <rmurri>, Project <default>, Status <DONE>, Queue <pub.1h>,
Command <sleep 60>, Share group charged </lsf_biol_all/lsf
_aeber/rmurri>
Mon Oct 8 17:07:54: Submitted from host <brutus4>, CWD <$HOME>, Output File <l
sf.o%J>;
Mon Oct 8 17:08:44: Dispatched to <a3201>;
Mon Oct 8 17:09:51: Completed <done>.
Accounting information about this job:
Share group charged </lsf_biol_all/lsf_aeber/rmurri>
CPU_T WAIT TURNAROUND STATUS HOG_FACTOR MEM SWAP
0.08 50 117 done 0.0007 5M 222M
------------------------------------------------------------------------------
SUMMARY: ( time unit: second )
Total number of done jobs: 1 Total number of exited jobs: 0
Total CPU time consumed: 0.1 Average CPU time consumed: 0.1
Maximum CPU time of a job: 0.1 Minimum CPU time of a job: 0.1
Total wait time in queues: 50.0
Average wait time in queue: 50.0
Maximum wait time in queue: 50.0 Minimum wait time in queue: 50.0
Average turnaround time: 117 (seconds/job)
Maximum turnaround time: 117 Minimum turnaround time: 117
Average hog factor of a job: 0.00 ( cpu time / turnaround time )
Maximum hog factor of a job: 0.00 Minimum hog factor of a job: 0.00
""",
# STDERR
'')
assert acct['duration'] == Duration('67s')
assert acct['used_cpu_time'] == Duration('0.08s')
assert acct['max_used_memory'] == Memory('227MB')
# timestamps
year = datetime.date.today().year
assert (acct['lsf_submission_time'] ==
datetime.datetime(year, 10, 8, 17, 7, 54))
assert (acct['lsf_start_time'] ==
datetime.datetime(year, 10, 8, 17, 8, 44))
assert (acct['lsf_completion_time'] ==
datetime.datetime(year, 10, 8, 17, 9, 51))