Exemple #1
0
def test_bacct_killed():
    """Test parsing `bacct -l` output for a canceled job."""
    lsf = LsfLrms(
        name='test',
        architecture=gc3libs.Run.Arch.X86_64,
        max_cores=1,
        max_cores_per_job=1,
        max_memory_per_core=1 * GB,
        max_walltime=1 * hours,
        auth=None,  # ignored if `transport` is `local`
        frontend='localhost',
        transport='local',
        bacct='bacct')
    acct = lsf._parse_acct_output("""
Accounting information about jobs that are:
  - submitted by all users.
  - accounted on all projects.
  - completed normally or exited
  - executed on all hosts.
  - submitted to all queues.
  - accounted on all service classes.
------------------------------------------------------------------------------

Job <3224113>, User <rmurri>, Project <default>, Status <EXIT>, Queue <pub.1h>,
                     Command <sleep 300>, Share group charged </lsf_biol_all/ls
                     f_aeber/rmurri>
Fri Oct  5 17:49:35: Submitted from host <brutus4>, CWD <$HOME>, Output File <l
                     sf.o%J>;
Fri Oct  5 17:50:35: Dispatched to <a3191>;
Fri Oct  5 17:51:30: Completed <exit>; TERM_OWNER: job killed by owner.

Accounting information about this job:
     Share group charged </lsf_biol_all/lsf_aeber/rmurri>
     CPU_T     WAIT     TURNAROUND   STATUS     HOG_FACTOR    MEM    SWAP
      0.04       60            115     exit         0.0003     1M     34M
------------------------------------------------------------------------------

SUMMARY:      ( time unit: second )
 Total number of done jobs:       0      Total number of exited jobs:     1
 Total CPU time consumed:       0.0      Average CPU time consumed:     0.0
 Maximum CPU time of a job:     0.0      Minimum CPU time of a job:     0.0
 Total wait time in queues:    60.0
 Average wait time in queue:   60.0
 Maximum wait time in queue:   60.0      Minimum wait time in queue:   60.0
 Average turnaround time:       115 (seconds/job)
 Maximum turnaround time:       115      Minimum turnaround time:       115
 Average hog factor of a job:  0.00 ( cpu time / turnaround time )
 Maximum hog factor of a job:  0.00      Minimum hog factor of a job:  0.00
""")
    assert_equal(acct['duration'], Duration('55s'))
    assert_equal(acct['used_cpu_time'], Duration('0.04s'))
    assert_equal(acct['max_used_memory'], Memory('35MB'))
    # timestamps
    year = datetime.date.today().year
    assert_equal(acct['lsf_submission_time'],
                 datetime.datetime(year, 10, 5, 17, 49, 35))
    assert_equal(acct['lsf_start_time'],
                 datetime.datetime(year, 10, 5, 17, 50, 35))
    assert_equal(acct['lsf_completion_time'],
                 datetime.datetime(year, 10, 5, 17, 51, 30))
Exemple #2
0
def test_bacct_done1():
    """Test parsing `bacct -l` output for a not-so-trivial job."""
    lsf = LsfLrms(
        name='test',
        architecture=gc3libs.Run.Arch.X86_64,
        max_cores=1,
        max_cores_per_job=1,
        max_memory_per_core=1 * GB,
        max_walltime=1 * hours,
        auth=None,  # ignored if `transport` is `local`
        frontend='localhost',
        transport='local',
        bacct='bacct')
    acct = lsf._parse_acct_output("""
Accounting information about jobs that are:
  - submitted by all users.
  - accounted on all projects.
  - completed normally or exited
  - executed on all hosts.
  - submitted to all queues.
  - accounted on all service classes.
------------------------------------------------------------------------------

Job <3329618>, User <rmurri>, Project <default>, Status <DONE>, Queue <pub.1h>,
                     Command <md5sum lsf.o3224113 lsf.o3224132>, Share group ch
                     arged </lsf_biol_all/lsf_aeber/rmurri>
Mon Oct  8 17:08:54: Submitted from host <brutus4>, CWD <$HOME>, Output File <l
                     sf.o%J>;
Mon Oct  8 17:10:01: Dispatched to <a3041>;
Mon Oct  8 17:10:07: Completed <done>.

Accounting information about this job:
     Share group charged </lsf_biol_all/lsf_aeber/rmurri>
     CPU_T     WAIT     TURNAROUND   STATUS     HOG_FACTOR    MEM    SWAP
      0.04       67             73     done         0.0005     3M     34M
------------------------------------------------------------------------------

SUMMARY:      ( time unit: second )
 Total number of done jobs:       1      Total number of exited jobs:     0
 Total CPU time consumed:       0.0      Average CPU time consumed:     0.0
 Maximum CPU time of a job:     0.0      Minimum CPU time of a job:     0.0
 Total wait time in queues:    67.0
 Average wait time in queue:   67.0
 Maximum wait time in queue:   67.0      Minimum wait time in queue:   67.0
 Average turnaround time:        73 (seconds/job)
 Maximum turnaround time:        73      Minimum turnaround time:        73
 Average hog factor of a job:  0.00 ( cpu time / turnaround time )
 Maximum hog factor of a job:  0.00      Minimum hog factor of a job:  0.00
    """)
    assert_equal(acct['duration'], Duration('6s'))
    assert_equal(acct['used_cpu_time'], Duration('0.04s'))
    assert_equal(acct['max_used_memory'], Memory('37MB'))
    # timestamps
    year = datetime.date.today().year
    assert_equal(acct['lsf_submission_time'],
                 datetime.datetime(year, 10, 8, 17, 8, 54))
    assert_equal(acct['lsf_start_time'],
                 datetime.datetime(year, 10, 8, 17, 10, 1))
    assert_equal(acct['lsf_completion_time'],
                 datetime.datetime(year, 10, 8, 17, 10, 7))
Exemple #3
0
def test_bacct_killed():
    """Test parsing `bacct -l` output for a canceled job."""
    lsf = LsfLrms(name='test',
                  architecture=gc3libs.Run.Arch.X86_64,
                  max_cores=1,
                  max_cores_per_job=1,
                  max_memory_per_core=1 * GB,
                  max_walltime=1 * hours,
                  auth=None,  # ignored if `transport` is `local`
                  frontend='localhost',
                  transport='local',
                  bacct='bacct')
    acct = lsf._parse_acct_output("""
Accounting information about jobs that are:
  - submitted by all users.
  - accounted on all projects.
  - completed normally or exited
  - executed on all hosts.
  - submitted to all queues.
  - accounted on all service classes.
------------------------------------------------------------------------------

Job <3224113>, User <rmurri>, Project <default>, Status <EXIT>, Queue <pub.1h>,
                     Command <sleep 300>, Share group charged </lsf_biol_all/ls
                     f_aeber/rmurri>
Fri Oct  5 17:49:35: Submitted from host <brutus4>, CWD <$HOME>, Output File <l
                     sf.o%J>;
Fri Oct  5 17:50:35: Dispatched to <a3191>;
Fri Oct  5 17:51:30: Completed <exit>; TERM_OWNER: job killed by owner.

Accounting information about this job:
     Share group charged </lsf_biol_all/lsf_aeber/rmurri>
     CPU_T     WAIT     TURNAROUND   STATUS     HOG_FACTOR    MEM    SWAP
      0.04       60            115     exit         0.0003     1M     34M
------------------------------------------------------------------------------

SUMMARY:      ( time unit: second )
 Total number of done jobs:       0      Total number of exited jobs:     1
 Total CPU time consumed:       0.0      Average CPU time consumed:     0.0
 Maximum CPU time of a job:     0.0      Minimum CPU time of a job:     0.0
 Total wait time in queues:    60.0
 Average wait time in queue:   60.0
 Maximum wait time in queue:   60.0      Minimum wait time in queue:   60.0
 Average turnaround time:       115 (seconds/job)
 Maximum turnaround time:       115      Minimum turnaround time:       115
 Average hog factor of a job:  0.00 ( cpu time / turnaround time )
 Maximum hog factor of a job:  0.00      Minimum hog factor of a job:  0.00
""")
    assert_equal(acct['duration'], Duration('55s'))
    assert_equal(acct['used_cpu_time'], Duration('0.04s'))
    assert_equal(acct['max_used_memory'], Memory('35MB'))
    # timestamps
    year = datetime.date.today().year
    assert_equal(acct['lsf_submission_time'],
                 datetime.datetime(year, 10, 5, 17, 49, 35))
    assert_equal(acct['lsf_start_time'],
                 datetime.datetime(year, 10, 5, 17, 50, 35))
    assert_equal(acct['lsf_completion_time'],
                 datetime.datetime(year, 10, 5, 17, 51, 30))
Exemple #4
0
def test_bacct_done1():
    """Test parsing `bacct -l` output for a not-so-trivial job."""
    lsf = LsfLrms(name='test',
                  architecture=gc3libs.Run.Arch.X86_64,
                  max_cores=1,
                  max_cores_per_job=1,
                  max_memory_per_core=1 * GB,
                  max_walltime=1 * hours,
                  auth=None,  # ignored if `transport` is `local`
                  frontend='localhost',
                  transport='local',
                  bacct='bacct')
    acct = lsf._parse_acct_output("""
Accounting information about jobs that are:
  - submitted by all users.
  - accounted on all projects.
  - completed normally or exited
  - executed on all hosts.
  - submitted to all queues.
  - accounted on all service classes.
------------------------------------------------------------------------------

Job <3329618>, User <rmurri>, Project <default>, Status <DONE>, Queue <pub.1h>,
                     Command <md5sum lsf.o3224113 lsf.o3224132>, Share group ch
                     arged </lsf_biol_all/lsf_aeber/rmurri>
Mon Oct  8 17:08:54: Submitted from host <brutus4>, CWD <$HOME>, Output File <l
                     sf.o%J>;
Mon Oct  8 17:10:01: Dispatched to <a3041>;
Mon Oct  8 17:10:07: Completed <done>.

Accounting information about this job:
     Share group charged </lsf_biol_all/lsf_aeber/rmurri>
     CPU_T     WAIT     TURNAROUND   STATUS     HOG_FACTOR    MEM    SWAP
      0.04       67             73     done         0.0005     3M     34M
------------------------------------------------------------------------------

SUMMARY:      ( time unit: second )
 Total number of done jobs:       1      Total number of exited jobs:     0
 Total CPU time consumed:       0.0      Average CPU time consumed:     0.0
 Maximum CPU time of a job:     0.0      Minimum CPU time of a job:     0.0
 Total wait time in queues:    67.0
 Average wait time in queue:   67.0
 Maximum wait time in queue:   67.0      Minimum wait time in queue:   67.0
 Average turnaround time:        73 (seconds/job)
 Maximum turnaround time:        73      Minimum turnaround time:        73
 Average hog factor of a job:  0.00 ( cpu time / turnaround time )
 Maximum hog factor of a job:  0.00      Minimum hog factor of a job:  0.00
    """)
    assert_equal(acct['duration'], Duration('6s'))
    assert_equal(acct['used_cpu_time'], Duration('0.04s'))
    assert_equal(acct['max_used_memory'], Memory('37MB'))
    # timestamps
    year = datetime.date.today().year
    assert_equal(acct['lsf_submission_time'],
                 datetime.datetime(year, 10, 8, 17, 8, 54))
    assert_equal(acct['lsf_start_time'],
                 datetime.datetime(year, 10, 8, 17, 10, 1))
    assert_equal(acct['lsf_completion_time'],
                 datetime.datetime(year, 10, 8, 17, 10, 7))
Exemple #5
0
def test_bjobs_output_for_accounting():
    lsf = LsfLrms(
        name='test',
        architecture=gc3libs.Run.Arch.X86_64,
        max_cores=1,
        max_cores_per_job=1,
        max_memory_per_core=1 * GB,
        max_walltime=1 * hours,
        auth=None,  # ignored if `transport` is `local`
        frontend='localhost',
        transport='local')
    bjobs_output = """
Job <131851>, Job Name <ChromaExtractShort>, User <wwolski>, Project <default>,
                     Status <DONE>, Queue <pub.8h>, Job Priority <50>, Command
                     <ChromatogramExtractor -in /cluster/scratch/malars/openswa
                     th/data/AQUA_fixed_water/split_napedro_L120224_001_SW-400A
                     QUA_no_background_2ul_dilution_10/split_napedro_L120224_00
                     1_SW-400AQUA_no_background_2ul_dilution_10_28.mzML.gz -tr
                     /cluster/scratch/malars/openswath/assays/iRT/DIA_iRT.TraML
                      -out split_napedro_L120224_001_SW-400AQUA_no_background_2
                     ul_dilution_10_28._rtnorm.chrom.mzML -is_swath -min_upper_
                     edge_dist 1 -threads 2>, Share group charged </lsf_biol_al
                     l/lsf_biol_other/wwolski>
Tue Jul 24 10:03:15: Submitted from host <brutus3>, CWD <$HOME/.gc3pie_jobs/lrm
                     s_job.YNZmU17755/.>, Output File <lsf.o%J>, Requested Reso
                     urces <select[mem<70000 && lustre] order[-ut] rusage[mem=1
                     000,m=1]>, Login Shell </bin/sh>, Specified Hosts <thin+9>
                     , <single+8>, <smp16+6>, <smp24+5>, <smp48+4>;

 RUNLIMIT
 480.0 min of a6122
Tue Jul 24 10:04:19: Started on <a6122>, Execution Home </cluster/home/biol/wwo
                     lski>, Execution CWD </cluster/home/biol/wwolski/.gc3pie_j
                     obs/lrms_job.YNZmU17755/.>;
Tue Jul 24 10:05:45: Done successfully. The CPU time used is 2.1 seconds.

 MEMORY USAGE:
 MAX MEM: 41 Mbytes;  AVG MEM: 41 Mbytes

 SCHEDULING PARAMETERS:
           r15s   r1m  r15m   ut      pg    io   ls    it    tmp    swp    mem
 loadSched   -     -     -     -       -     -    -     -  1000M     -      -
 loadStop    -     -     -     -       -     -    -     -     -      -      -

          scratch      xs       s       m       l      xl      sp
 loadSched 4000.0      -       -       -       -       -       -
 loadStop      -       -       -       -       -       -       -
"""

    # Also parse the output of jobs to get accounting information
    acct = lsf._parse_acct_output(bjobs_output)
    assert_equal(acct['duration'], Duration('86s'))
    assert_equal(acct['used_cpu_time'], Duration('2.1s'))
    assert_equal(acct['max_used_memory'], Memory('41MB'))
Exemple #6
0
def test_bjobs_output_for_accounting():
    lsf = LsfLrms(name='test',
                  architecture=gc3libs.Run.Arch.X86_64,
                  max_cores=1,
                  max_cores_per_job=1,
                  max_memory_per_core=1 * GB,
                  max_walltime=1 * hours,
                  auth=None,  # ignored if `transport` is `local`
                  frontend='localhost',
                  transport='local')
    bjobs_output = """
Job <131851>, Job Name <ChromaExtractShort>, User <wwolski>, Project <default>,
                     Status <DONE>, Queue <pub.8h>, Job Priority <50>, Command
                     <ChromatogramExtractor -in /cluster/scratch/malars/openswa
                     th/data/AQUA_fixed_water/split_napedro_L120224_001_SW-400A
                     QUA_no_background_2ul_dilution_10/split_napedro_L120224_00
                     1_SW-400AQUA_no_background_2ul_dilution_10_28.mzML.gz -tr
                     /cluster/scratch/malars/openswath/assays/iRT/DIA_iRT.TraML
                      -out split_napedro_L120224_001_SW-400AQUA_no_background_2
                     ul_dilution_10_28._rtnorm.chrom.mzML -is_swath -min_upper_
                     edge_dist 1 -threads 2>, Share group charged </lsf_biol_al
                     l/lsf_biol_other/wwolski>
Tue Jul 24 10:03:15: Submitted from host <brutus3>, CWD <$HOME/.gc3pie_jobs/lrm
                     s_job.YNZmU17755/.>, Output File <lsf.o%J>, Requested Reso
                     urces <select[mem<70000 && lustre] order[-ut] rusage[mem=1
                     000,m=1]>, Login Shell </bin/sh>, Specified Hosts <thin+9>
                     , <single+8>, <smp16+6>, <smp24+5>, <smp48+4>;

 RUNLIMIT
 480.0 min of a6122
Tue Jul 24 10:04:19: Started on <a6122>, Execution Home </cluster/home/biol/wwo
                     lski>, Execution CWD </cluster/home/biol/wwolski/.gc3pie_j
                     obs/lrms_job.YNZmU17755/.>;
Tue Jul 24 10:05:45: Done successfully. The CPU time used is 2.1 seconds.

 MEMORY USAGE:
 MAX MEM: 41 Mbytes;  AVG MEM: 41 Mbytes

 SCHEDULING PARAMETERS:
           r15s   r1m  r15m   ut      pg    io   ls    it    tmp    swp    mem
 loadSched   -     -     -     -       -     -    -     -  1000M     -      -
 loadStop    -     -     -     -       -     -    -     -     -      -      -

          scratch      xs       s       m       l      xl      sp
 loadSched 4000.0      -       -       -       -       -       -
 loadStop      -       -       -       -       -       -       -
"""

    # Also parse the output of jobs to get accounting information
    acct = lsf._parse_acct_output(bjobs_output, '')
    assert_equal(acct['duration'], Duration('86s'))
    assert_equal(acct['used_cpu_time'], Duration('2.1s'))
    assert_equal(acct['max_used_memory'], Memory('41MB'))
Exemple #7
0
def test_bacct_done0():
    """Test parsing accounting information of a <sleep 300> job."""
    # gotten with `bacct -l "jobid"`
    acct = LsfLrms._parse_acct_output("""
Accounting information about jobs that are:
  - submitted by all users.
  - accounted on all projects.
  - completed normally or exited
  - executed on all hosts.
  - submitted to all queues.
  - accounted on all service classes.
------------------------------------------------------------------------------

Job <3329613>, User <rmurri>, Project <default>, Status <DONE>, Queue <pub.1h>,
                     Command <sleep 60>, Share group charged </lsf_biol_all/lsf
                     _aeber/rmurri>
Mon Oct  8 17:07:54: Submitted from host <brutus4>, CWD <$HOME>, Output File <l
                     sf.o%J>;
Mon Oct  8 17:08:44: Dispatched to <a3201>;
Mon Oct  8 17:09:51: Completed <done>.

Accounting information about this job:
     Share group charged </lsf_biol_all/lsf_aeber/rmurri>
     CPU_T     WAIT     TURNAROUND   STATUS     HOG_FACTOR    MEM    SWAP
      0.08       50            117     done         0.0007     5M    222M
------------------------------------------------------------------------------

SUMMARY:      ( time unit: second )
 Total number of done jobs:       1      Total number of exited jobs:     0
 Total CPU time consumed:       0.1      Average CPU time consumed:     0.1
 Maximum CPU time of a job:     0.1      Minimum CPU time of a job:     0.1
 Total wait time in queues:    50.0
 Average wait time in queue:   50.0
 Maximum wait time in queue:   50.0      Minimum wait time in queue:   50.0
 Average turnaround time:       117 (seconds/job)
 Maximum turnaround time:       117      Minimum turnaround time:       117
 Average hog factor of a job:  0.00 ( cpu time / turnaround time )
 Maximum hog factor of a job:  0.00      Minimum hog factor of a job:  0.00

    """)
    assert_equal(acct['duration'], Duration('67s'))
    assert_equal(acct['used_cpu_time'], Duration('0.08s'))
    assert_equal(acct['max_used_memory'], Memory('227MB'))
    # timestamps
    year = datetime.date.today().year
    assert_equal(acct['lsf_submission_time'], datetime.datetime(year, 10, 8, 17, 7, 54))
    assert_equal(acct['lsf_start_time'],      datetime.datetime(year, 10, 8, 17, 8, 44))
    assert_equal(acct['lsf_completion_time'], datetime.datetime(year, 10, 8, 17, 9, 51))
Exemple #8
0
def test_bacct_done0():
    """Test parsing accounting information of a <sleep 300> job."""
    # gotten with `bacct -l "jobid"`
    lsf = LsfLrms(name='test',
                  architecture=gc3libs.Run.Arch.X86_64,
                  max_cores=1,
                  max_cores_per_job=1,
                  max_memory_per_core=1 * GB,
                  max_walltime=1 * hours,
                  auth=None,  # ignored if `transport` is `local`
                  frontend='localhost',
                  transport='local',
                  bacct='bacct')
    acct = lsf._parse_acct_output("""
Accounting information about jobs that are:
  - submitted by all users.
  - accounted on all projects.
  - completed normally or exited
  - executed on all hosts.
  - submitted to all queues.
  - accounted on all service classes.
------------------------------------------------------------------------------

Job <3329613>, User <rmurri>, Project <default>, Status <DONE>, Queue <pub.1h>,
                     Command <sleep 60>, Share group charged </lsf_biol_all/lsf
                     _aeber/rmurri>
Mon Oct  8 17:07:54: Submitted from host <brutus4>, CWD <$HOME>, Output File <l
                     sf.o%J>;
Mon Oct  8 17:08:44: Dispatched to <a3201>;
Mon Oct  8 17:09:51: Completed <done>.

Accounting information about this job:
     Share group charged </lsf_biol_all/lsf_aeber/rmurri>
     CPU_T     WAIT     TURNAROUND   STATUS     HOG_FACTOR    MEM    SWAP
      0.08       50            117     done         0.0007     5M    222M
------------------------------------------------------------------------------

SUMMARY:      ( time unit: second )
 Total number of done jobs:       1      Total number of exited jobs:     0
 Total CPU time consumed:       0.1      Average CPU time consumed:     0.1
 Maximum CPU time of a job:     0.1      Minimum CPU time of a job:     0.1
 Total wait time in queues:    50.0
 Average wait time in queue:   50.0
 Maximum wait time in queue:   50.0      Minimum wait time in queue:   50.0
 Average turnaround time:       117 (seconds/job)
 Maximum turnaround time:       117      Minimum turnaround time:       117
 Average hog factor of a job:  0.00 ( cpu time / turnaround time )
 Maximum hog factor of a job:  0.00      Minimum hog factor of a job:  0.00

    """,
    # STDERR
    '')
    assert acct['duration'] == Duration('67s')
    assert acct['used_cpu_time'] == Duration('0.08s')
    assert acct['max_used_memory'] == Memory('227MB')
    # timestamps
    year = datetime.date.today().year
    assert (acct['lsf_submission_time'] ==
                 datetime.datetime(year, 10, 8, 17, 7, 54))
    assert (acct['lsf_start_time'] ==
                 datetime.datetime(year, 10, 8, 17, 8, 44))
    assert (acct['lsf_completion_time'] ==
                 datetime.datetime(year, 10, 8, 17, 9, 51))