Пример #1
0
    def test_non_mpi(self):
        """Test that node and cpu count are set to zero if MPI isn't used."""
        parser = LSFParser('testSite', 'testHost', False)

        line = ('"JOB_FINISH" "5.1" 1089407406 699195 283 33554482 1 1089290023'
                ' 0 0 1089406862 "raortega" "8nm" "" "" "" "lxplus015" "prog/st'
                'ep3c" "" "/af/step3c-362.txt" "/af/epc-362.txt" "1089290023.69'
                '9195" 0 1 "tbed0079" 64 3.3 "" "/af/hset.pl 362 7 8" 277.2100 '
                '17.280000 0 0 -1 0 0 927804 87722 0 0 0 -1 0 0 0 0 0 -1 "" "de'
                'fault" 0 1 "" "" 0 310424 339112 "" "" ""')

        record = parser.parse(line)
        cont = record._record_content

        self.assertEquals(cont['NodeCount'], 0,
                          "Node count not zero for non-mpi parser")
        self.assertEquals(cont['Processors'], 0,
                          "Processors not zero for non-mpi parser")
Пример #2
0
    def test_non_mpi(self):
        """Test that node and cpu count are set to zero if MPI isn't used."""
        parser = LSFParser('testSite', 'testHost', False)

        line = (
            '"JOB_FINISH" "5.1" 1089407406 699195 283 33554482 1 1089290023'
            ' 0 0 1089406862 "raortega" "8nm" "" "" "" "lxplus015" "prog/st'
            'ep3c" "" "/af/step3c-362.txt" "/af/epc-362.txt" "1089290023.69'
            '9195" 0 1 "tbed0079" 64 3.3 "" "/af/hset.pl 362 7 8" 277.2100 '
            '17.280000 0 0 -1 0 0 927804 87722 0 0 0 -1 0 0 0 0 0 -1 "" "de'
            'fault" 0 1 "" "" 0 310424 339112 "" "" ""')

        record = parser.parse(line)
        cont = record._record_content

        self.assertEquals(cont['NodeCount'], 0,
                          "Node count not zero for non-mpi parser")
        self.assertEquals(cont['Processors'], 0,
                          "Processors not zero for non-mpi parser")
Пример #3
0
class ParserLSFTest(unittest.TestCase):
    '''
    Test case for LSF parser
    '''
    def setUp(self):
        self.parser = LSFParser('testSite', 'testHost', True)

    def test_parse(self):

        fields = ('JobName', 'LocalUserID', 'LocalUserGroup', 'WallDuration',
                  'CpuDuration', 'StartTime', 'StopTime', 'MemoryReal',
                  'MemoryVirtual', 'NodeCount', 'Processors')

        lines = (
            ('"JOB_FINISH" "5.1" 1089407406 699195 283 33554482 1 1089290023 0 0 1089406862 '
             '"raortega" "8nm" "" "" "" "lxplus015" "prog/step3c" "" "/afs/cern.ch/user/r/raortega/log/bstep3c-362.txt" '
             '"/afs/cern.ch/user/r/raortega/log/berr-step3c-362.txt" "1089290023.699195" 0 1 "tbed0079" 64 3.3 "" '
             '"/afs/cern.ch/user/r/raortega/prog/step3c/startEachset.pl 362 7 8" 277.210000 17.280000 0 0 -1 0 0 927804'
             ' 87722 0 0 0 -1 0 0 0 0 0 -1 "" "default" 0 1 "" "" 0 310424 339112 "" "" ""'
             ),
            # MPI information
            ('"JOB_FINISH" "7.06" 1302184020 436491 10001 33816691 16 1302184001 0 0 1302184004 "viglen" "normal" '
             '" span[ptile=8] cu[maxcus=1]" "" "" "comp000" "benchmarks/IMB_3.0/bin/bsub" "" "IMB.%J.%I.out" '
             '"IMB.%J.%I.err" "1302184001.436491" 0 16 "comp042" "comp042" "comp042" "comp042" "comp042" "comp042" '
             '"comp042" "comp042" "comp043" "comp043" "comp043" "comp043" "comp043" "comp043" "comp043" "comp043" 32'
             ' 60.0 "imb_sr[1-90]" "#!/bin/bash; #BSUB -n 16;#BSUB -R \'span[ptile=8]\';#BSUB -R \'cu[maxcus=1]\';#BSUB '
             '-x;#BSUB -J imb_sr[1-90];####BSUB -m ""comp000 comp001"";#BSUB -o IMB.%J.%I.out;#BSUB -e IMB.%J.%I.err; '
             '# Output some env vars;echo ""LSB_HOSTS: $LSB_HOSTS"";echo ""LSB_MCPU_HOSTS: $LSB_MCPU_HOSTS"";echo '
             '""LSB_DJOB_HOSTFILE: $LSB_DJOB_HOSTFILE"";echo ""LSB_DJOB_HOSTFILE (cat):""; cat $LSB_DJOB_HOSTFILE; '
             'HOSTLIST=`cat $LSB_DJOB_HOSTFILE | uniq | tr ""\n"" "" ""`; . /etc/profile; module add MPI/PMPI;cd '
             '/home/test/viglen/benchmarks/IMB_3.0/bin/ ;#mpirun -np $LSB_DJOB_NUMPROC ./imb-pmpi -msglen ./IMB_msglen '
             '-map 2x8 -npmin 16 sendrecv;#mpirun -d -prot -np $LSB_DJOB_NUMPROC -hostlist ""comp000 comp001"" '
             './imb-pmpi -msglen ./IMB_msglen -map 2x8 -npmin 16 sendrecv;mpirun -d -prot -np $LSB_DJOB_NUMPROC '
             '-hostlist ""$HOSTLIST"" ./imb-pmpi -msglen ./IMB_msglen -map 2x8 -npmin 16 sendrecv" 0.472928 1.112830 '
             '0 0 -1 0 0 149746 0 0 0 0 -1 0 0 0 1907 1746 -1 "" "default" 65280 16 "" "" 23 2 25 "" "" "" "" 0 "" 0 ""'
             ' -1 "/viglen" "" "default" "" -1 "" "" 6160  "" 1302184004 "" "" 0'
             ),
            # LSF 9 log (Mixed quotes used to handle mixed quotes in string)
            ('"JOB_FINISH" "9.11" 1380043655 31255091 386919 36175899 1 1380043614 0 0 1380043630 "oops17" "abcgrid" '
             '"" "" "" "nice.phys.abc-uni.uk" "" "/dev/null" "/dev/null" "" "191/1380043614.31255091" 0 1 "a0052" 64 '
             '78.0 "cream_123456789" "#!/bin/bash;# LSF job wrapper generated by lsf_submit.sh;# on Tue Sep 24 19:2'
             '6:51 CEST 2013;#;# LSF directives:;#BSUB -L /bin/bash;#BSUB -J cream_123456789;#BSUB -q abcgrid;#BSUB -n'
             '1;#BSUB -f ""/var/cream_sandbox/ops/CN_Robot__grid_client___Someone_Somewhere'
             'NGI_Somewhere_Role_NULL_Capability_NULL_ops017/18/CREAM123456789/CREAM123456789_jobWrapper.sh > CREAM18173'
             '0262_jobWrapper.sh.386919.7105.1380043611"";#BSUB -f ""/var/cream_sandbox/ops/CN_Robot__grid_client___So'
             'meone_Somewhere/proxy/426e0fbced998f1b9162bb6a0db0eeedc2c_18361079272219 > cream_123456789.proxy"";#BSUB'
             '-f ""/var/cream_sandbox/ops/CN_Robot__grid_client___Someone_Somewhere/18/CREAM123456789/StandardOutput <'
             'out_cream_123456789_StandardOutput"";#BSUB -f ""/var/cream_sandbox/ops/CN_Robot__grid_client___Someone_S'
             'omewhere/18/CREAM123456789/StandardError < err_cream_181730262_StandardError""; # Check whet'
             'her we need to move to the LSF original CWD:;if [ -d ""$CERN_STARTER_ORIGINAL_CWD"" ]; then;    cd $CERN'
             '_STARTER_ORIGINAL_CWD;fi;old_home=`pwd`;new_home=${old_home}/home_cream_123456789;mkdir $new_home;trap '
             "'wait $job_pid; cd $old_home; rm -rf $new_home; exit 255' 1 2 3 15 24;trap 'wait $job_pid; cd $old_home;"
             "rm -rf $new_home' 0;# Copy into new home any shared input sandbox file;# Move into new home any relative"
             'input sandbox file;mv ""CREAM123456789_jobWrapper.sh.386919.7105.1380043611"" ""$new_home/CREAM123456789'
             '_jobWrapper.sh"" &> /dev/null;mv ""cream_123456789.proxy"" ""$new_home/cream_123456789.proxy"" &> /dev/n'
             'ull;export HOME=$new_home;cd $new_home;# Resetting proxy to local position;export X509_USER_PROXY=$new_h'
             'ome/cream_123456789.proxy; # Command to execute:;if [ ! -x ./CREAM123456789_jobWrapper.sh ]; then chmod '
             'u+x ./CREAM123456789_jobWrapper.sh; fi;if [ -x ${GLITE_LOCATION:-/opt/glite}/libexec/jobwrapper ];then;$'
             '{GLITE_LOCATION:-/opt/glite}/libexec/jobwrapper ./CREAM123456789_jobWrapper.sh  > ""out_cream_123456789_'
             'StandardOutput"" 2> ""err_cream_123456789_StandardError"" & elif [ -x /opt/lcg/libexec/jobwrapper ];then'
             ';/opt/lcg/libexec/jobwrapper ./CREAM123456789_jobWrapper.sh  > ""out_cream_123456789_StandardOutput"" 2>'
             '""err_cream_123456789_StandardError"" & elif [ -x $BLAH_AUX_JOBWRAPPER ];then;$BLAH_AUX_JOBWRAPPER ./CRE'
             'AM123456789_jobWrapper.sh  > ""out_cream_123456789_StandardOutput"" 2> ""err_cream_123456789_StandardErr'
             'or"" & else;$new_home/CREAM123456789_jobWrapper.sh  > ""out_cream_123456789_StandardOutput"" 2> ""err_cr'
             'eam_123456789_StandardError"" & fi;job_pid=$!; # Wait for the user job to finish;wait $job_pid;user_retc'
             'ode=$?; # Move all relative outputsand paths out of temp home;cd $new_home;mv ""out_cream_123456789_Stan'
             'dardOutput"" ""$old_home/out_cream_123456789_StandardOutput"" 2> /dev/null;mv ""err_cream_123456789_Stan'
             'dardError"" ""$old_home/err_cream_123456789_StandardError"" 2> /dev/null;# Move any remapped outputsand '
             'file to shared directories; # Remove the staged files, if any;rm ""CREAM123456789_jobWrapper.sh"" 2> /de'
             'v/null;rm ""cream_123456789.proxy"" 2> /dev/null;cd $old_home; exit $user_retcode" 15.093705 8.511706 52'
             '96 0 -1 0 0 26452 0 0 0 16 -1 0 0 0 242 171 -1 "" "default" 0 1 "/bin/bash" "" 0 1495304 0 "" "" "" "" 0'
             '"" 0 "" -1 "/ops/ops017" "" "" "" -1 "" "" 6160 "" 1380043630 "" "" 0 0 -1 0 441780 "select[ type == any'
             '] order[-ut] rusage[r1m=0.91:duration=2m:decay=0] " "" -1 "" -1 0 "" 0 0 "" 25 "" 0 ""'
             ))

        values = (('699195', 'raortega', None, 544, 294,
                   datetime.datetime.utcfromtimestamp(1089406862),
                   datetime.datetime.utcfromtimestamp(1089407406), 310424,
                   339112, 1, 1),
                  ('436491', 'viglen', None, 16, 2,
                   datetime.datetime.utcfromtimestamp(1302184004),
                   datetime.datetime.utcfromtimestamp(1302184020), 2, 25, 2,
                   16), ('31255091', 'oops17', None, 25, 24,
                         datetime.datetime.utcfromtimestamp(1380043630),
                         datetime.datetime.utcfromtimestamp(1380043655),
                         1495304, 0, 1, 1))

        cases = {}
        for line, value in zip(lines, values):
            cases[line] = dict(zip(fields, value))

        for line in cases.keys():

            record = self.parser.parse(line)
            cont = record._record_content

            self.assertEqual(cont['Site'], 'testSite')
            self.assertEqual(cont['MachineName'], 'testHost')
            self.assertEqual(cont['Infrastructure'], 'APEL-CREAM-LSF')

            for field in cases[line].keys():
                self.assertTrue(field in cont,
                                "Field '%s' not in record: %s" % (field, cont))

            for key in cases[line].keys():
                self.assertEqual(
                    cont[key], cases[line][key],
                    "%s != %s for key %s" % (cont[key], cases[line][key], key))

    def test_invalid_expr(self):
        # two fields are not separated by space
        line = (
            '"JOB_FINISH" "5.1" 1089407406 699195 283 33554482 1 1089290023 0 0 1089406862 '
            '"raortega" "8nm" "" "" "" "lxplus015" "prog/step3c" "" "/afs/cern.ch/user/r/raortega/log/bstep3c-362.txt" '
            '"/afs/cern.ch/user/r/raortega/log/berr-step3c-362.txt""1089290023.699195" 0 1 "tbed0079" 64 3.3 "" '
            '"/afs/cern.ch/user/r/raortega/prog/step3c/startEachset.pl362 7 8" 277.210000 17.280000 0 0 -1 0 0 927804'
            ' 87722 0 0 0 -1 0 0 0 0 0 -1 "" "default" 0 1 "" "" 0 310424 339112 "" "" ""'
        )

        self.assertRaises(IndexError, self.parser.parse, line)

    def test_wall_value_error(self):
        """Check that stop before start raises a ValueError."""
        line = (
            '"JOB_FINISH" "5.1" 1089407406 699195 283 33554482 1 1089290023'
            ' 0 0 1089409862 "raortega" "8nm" "" "" "" "lxplus015" "prog/st'
            'ep3c" "" "/step3c-362.txt" "/berr-step3c-362.txt" "1089290023.'
            '699195" 0 1 "tbed0079" 64 3.3 "" "/artEachset.j 362 7 8" 277.2'
            '10000 17.280000 0 0 -1 0 0 927804 87722 0 0 0 -1 0 0 0 0 0 -1 '
            '"" "default" 0 1 "" "" 0 310424 339112 "" "" ""')
        self.assertRaises(ValueError, self.parser.parse, line)

    def test_unfinished(self):
        """Check that a valid line without JOB_FINISH returns None."""
        line = (
            '"JOB_RESIZE" "5.1" 1089407406 699195 283 33554482 1 1089290023'
            ' 0 0 1089406862 "raortega" "8nm" "" "" "" "lxplus015" "prog/st'
            'ep3c" "" "/step3c-362.txt" "/berr-step3c-362.txt" "1089290023.'
            '699195" 0 1 "tbed0079" 64 3.3 "" "/artEachset.j 362 7 8" 277.2'
            '10000 17.280000 0 0 -1 0 0 927804 87722 0 0 0 -1 0 0 0 0 0 -1 '
            '"" "default" 0 1 "" "" 0 310424 339112 "" "" ""')

        self.assertEqual(self.parser.parse(line), None)

    def test_non_mpi(self):
        """Test that node and cpu count are set to zero if MPI isn't used."""
        parser = LSFParser('testSite', 'testHost', False)

        line = (
            '"JOB_FINISH" "5.1" 1089407406 699195 283 33554482 1 1089290023'
            ' 0 0 1089406862 "raortega" "8nm" "" "" "" "lxplus015" "prog/st'
            'ep3c" "" "/af/step3c-362.txt" "/af/epc-362.txt" "1089290023.69'
            '9195" 0 1 "tbed0079" 64 3.3 "" "/af/hset.pl 362 7 8" 277.2100 '
            '17.280000 0 0 -1 0 0 927804 87722 0 0 0 -1 0 0 0 0 0 -1 "" "de'
            'fault" 0 1 "" "" 0 310424 339112 "" "" ""')

        record = parser.parse(line)
        cont = record._record_content

        self.assertEquals(cont['NodeCount'], 0,
                          "Node count not zero for non-mpi parser")
        self.assertEquals(cont['Processors'], 0,
                          "Processors not zero for non-mpi parser")
Пример #4
0
class ParserLSFTest(unittest.TestCase):
    """
    Test case for LSF parser
    """

    def setUp(self):
        self.parser = LSFParser("testSite", "testHost", True)

    def test_parse(self):

        fields = (
            "JobName",
            "LocalUserID",
            "LocalUserGroup",
            "WallDuration",
            "CpuDuration",
            "StartTime",
            "StopTime",
            "MemoryReal",
            "MemoryVirtual",
            "NodeCount",
            "Processors",
        )

        lines = (
            (
                '"JOB_FINISH" "5.1" 1089407406 699195 283 33554482 1 1089290023 0 0 1089406862 '
                '"raortega" "8nm" "" "" "" "lxplus015" "prog/step3c" "" "/afs/cern.ch/user/r/raortega/log/bstep3c-362.txt" '
                '"/afs/cern.ch/user/r/raortega/log/berr-step3c-362.txt" "1089290023.699195" 0 1 "tbed0079" 64 3.3 "" '
                '"/afs/cern.ch/user/r/raortega/prog/step3c/startEachset.pl 362 7 8" 277.210000 17.280000 0 0 -1 0 0 927804'
                ' 87722 0 0 0 -1 0 0 0 0 0 -1 "" "default" 0 1 "" "" 0 310424 339112 "" "" ""'
            ),
            # MPI information
            (
                '"JOB_FINISH" "7.06" 1302184020 436491 10001 33816691 16 1302184001 0 0 1302184004 "viglen" "normal" '
                '" span[ptile=8] cu[maxcus=1]" "" "" "comp000" "benchmarks/IMB_3.0/bin/bsub" "" "IMB.%J.%I.out" '
                '"IMB.%J.%I.err" "1302184001.436491" 0 16 "comp042" "comp042" "comp042" "comp042" "comp042" "comp042" '
                '"comp042" "comp042" "comp043" "comp043" "comp043" "comp043" "comp043" "comp043" "comp043" "comp043" 32'
                " 60.0 \"imb_sr[1-90]\" \"#!/bin/bash; #BSUB -n 16;#BSUB -R 'span[ptile=8]';#BSUB -R 'cu[maxcus=1]';#BSUB "
                '-x;#BSUB -J imb_sr[1-90];####BSUB -m ""comp000 comp001"";#BSUB -o IMB.%J.%I.out;#BSUB -e IMB.%J.%I.err; '
                '# Output some env vars;echo ""LSB_HOSTS: $LSB_HOSTS"";echo ""LSB_MCPU_HOSTS: $LSB_MCPU_HOSTS"";echo '
                '""LSB_DJOB_HOSTFILE: $LSB_DJOB_HOSTFILE"";echo ""LSB_DJOB_HOSTFILE (cat):""; cat $LSB_DJOB_HOSTFILE; '
                'HOSTLIST=`cat $LSB_DJOB_HOSTFILE | uniq | tr ""\n"" "" ""`; . /etc/profile; module add MPI/PMPI;cd '
                "/home/test/viglen/benchmarks/IMB_3.0/bin/ ;#mpirun -np $LSB_DJOB_NUMPROC ./imb-pmpi -msglen ./IMB_msglen "
                '-map 2x8 -npmin 16 sendrecv;#mpirun -d -prot -np $LSB_DJOB_NUMPROC -hostlist ""comp000 comp001"" '
                "./imb-pmpi -msglen ./IMB_msglen -map 2x8 -npmin 16 sendrecv;mpirun -d -prot -np $LSB_DJOB_NUMPROC "
                '-hostlist ""$HOSTLIST"" ./imb-pmpi -msglen ./IMB_msglen -map 2x8 -npmin 16 sendrecv" 0.472928 1.112830 '
                '0 0 -1 0 0 149746 0 0 0 0 -1 0 0 0 1907 1746 -1 "" "default" 65280 16 "" "" 23 2 25 "" "" "" "" 0 "" 0 ""'
                ' -1 "/viglen" "" "default" "" -1 "" "" 6160  "" 1302184004 "" "" 0'
            ),
            # LSF 9 log (Mixed quotes used to handle mixed quotes in string)
            (
                '"JOB_FINISH" "9.11" 1380043655 31255091 386919 36175899 1 1380043614 0 0 1380043630 "oops17" "abcgrid" '
                '"" "" "" "nice.phys.abc-uni.uk" "" "/dev/null" "/dev/null" "" "191/1380043614.31255091" 0 1 "a0052" 64 '
                '78.0 "cream_123456789" "#!/bin/bash;# LSF job wrapper generated by lsf_submit.sh;# on Tue Sep 24 19:2'
                "6:51 CEST 2013;#;# LSF directives:;#BSUB -L /bin/bash;#BSUB -J cream_123456789;#BSUB -q abcgrid;#BSUB -n"
                '1;#BSUB -f ""/var/cream_sandbox/ops/CN_Robot__grid_client___Someone_Somewhere'
                "NGI_Somewhere_Role_NULL_Capability_NULL_ops017/18/CREAM123456789/CREAM123456789_jobWrapper.sh > CREAM18173"
                '0262_jobWrapper.sh.386919.7105.1380043611"";#BSUB -f ""/var/cream_sandbox/ops/CN_Robot__grid_client___So'
                'meone_Somewhere/proxy/426e0fbced998f1b9162bb6a0db0eeedc2c_18361079272219 > cream_123456789.proxy"";#BSUB'
                '-f ""/var/cream_sandbox/ops/CN_Robot__grid_client___Someone_Somewhere/18/CREAM123456789/StandardOutput <'
                'out_cream_123456789_StandardOutput"";#BSUB -f ""/var/cream_sandbox/ops/CN_Robot__grid_client___Someone_S'
                'omewhere/18/CREAM123456789/StandardError < err_cream_181730262_StandardError""; # Check whet'
                'her we need to move to the LSF original CWD:;if [ -d ""$CERN_STARTER_ORIGINAL_CWD"" ]; then;    cd $CERN'
                "_STARTER_ORIGINAL_CWD;fi;old_home=`pwd`;new_home=${old_home}/home_cream_123456789;mkdir $new_home;trap "
                "'wait $job_pid; cd $old_home; rm -rf $new_home; exit 255' 1 2 3 15 24;trap 'wait $job_pid; cd $old_home;"
                "rm -rf $new_home' 0;# Copy into new home any shared input sandbox file;# Move into new home any relative"
                'input sandbox file;mv ""CREAM123456789_jobWrapper.sh.386919.7105.1380043611"" ""$new_home/CREAM123456789'
                '_jobWrapper.sh"" &> /dev/null;mv ""cream_123456789.proxy"" ""$new_home/cream_123456789.proxy"" &> /dev/n'
                "ull;export HOME=$new_home;cd $new_home;# Resetting proxy to local position;export X509_USER_PROXY=$new_h"
                "ome/cream_123456789.proxy; # Command to execute:;if [ ! -x ./CREAM123456789_jobWrapper.sh ]; then chmod "
                "u+x ./CREAM123456789_jobWrapper.sh; fi;if [ -x ${GLITE_LOCATION:-/opt/glite}/libexec/jobwrapper ];then;$"
                '{GLITE_LOCATION:-/opt/glite}/libexec/jobwrapper ./CREAM123456789_jobWrapper.sh  > ""out_cream_123456789_'
                'StandardOutput"" 2> ""err_cream_123456789_StandardError"" & elif [ -x /opt/lcg/libexec/jobwrapper ];then'
                ';/opt/lcg/libexec/jobwrapper ./CREAM123456789_jobWrapper.sh  > ""out_cream_123456789_StandardOutput"" 2>'
                '""err_cream_123456789_StandardError"" & elif [ -x $BLAH_AUX_JOBWRAPPER ];then;$BLAH_AUX_JOBWRAPPER ./CRE'
                'AM123456789_jobWrapper.sh  > ""out_cream_123456789_StandardOutput"" 2> ""err_cream_123456789_StandardErr'
                'or"" & else;$new_home/CREAM123456789_jobWrapper.sh  > ""out_cream_123456789_StandardOutput"" 2> ""err_cr'
                'eam_123456789_StandardError"" & fi;job_pid=$!; # Wait for the user job to finish;wait $job_pid;user_retc'
                'ode=$?; # Move all relative outputsand paths out of temp home;cd $new_home;mv ""out_cream_123456789_Stan'
                'dardOutput"" ""$old_home/out_cream_123456789_StandardOutput"" 2> /dev/null;mv ""err_cream_123456789_Stan'
                'dardError"" ""$old_home/err_cream_123456789_StandardError"" 2> /dev/null;# Move any remapped outputsand '
                'file to shared directories; # Remove the staged files, if any;rm ""CREAM123456789_jobWrapper.sh"" 2> /de'
                'v/null;rm ""cream_123456789.proxy"" 2> /dev/null;cd $old_home; exit $user_retcode" 15.093705 8.511706 52'
                '96 0 -1 0 0 26452 0 0 0 16 -1 0 0 0 242 171 -1 "" "default" 0 1 "/bin/bash" "" 0 1495304 0 "" "" "" "" 0'
                '"" 0 "" -1 "/ops/ops017" "" "" "" -1 "" "" 6160 "" 1380043630 "" "" 0 0 -1 0 441780 "select[ type == any'
                '] order[-ut] rusage[r1m=0.91:duration=2m:decay=0] " "" -1 "" -1 0 "" 0 0 "" 25 "" 0 ""'
            ),
        )

        values = (
            (
                "699195",
                "raortega",
                None,
                544,
                294,
                datetime.datetime.utcfromtimestamp(1089406862),
                datetime.datetime.utcfromtimestamp(1089407406),
                310424,
                339112,
                1,
                1,
            ),
            (
                "436491",
                "viglen",
                None,
                16,
                2,
                datetime.datetime.utcfromtimestamp(1302184004),
                datetime.datetime.utcfromtimestamp(1302184020),
                2,
                25,
                2,
                16,
            ),
            (
                "31255091",
                "oops17",
                None,
                25,
                24,
                datetime.datetime.utcfromtimestamp(1380043630),
                datetime.datetime.utcfromtimestamp(1380043655),
                1495304,
                0,
                1,
                1,
            ),
        )

        cases = {}
        for line, value in zip(lines, values):
            cases[line] = dict(zip(fields, value))

        for line in cases.keys():

            record = self.parser.parse(line)
            cont = record._record_content

            self.assertEqual(cont["Site"], "testSite")
            self.assertEqual(cont["MachineName"], "testHost")
            self.assertEqual(cont["Infrastructure"], "APEL-CREAM-LSF")

            for field in cases[line].keys():
                self.assertTrue(field in cont, "Field '%s' not in record: %s" % (field, cont))

            for key in cases[line].keys():
                self.assertEqual(
                    cont[key], cases[line][key], "%s != %s for key %s" % (cont[key], cases[line][key], key)
                )

    def test_invalid_expr(self):
        # two fields are not separated by space
        line = (
            '"JOB_FINISH" "5.1" 1089407406 699195 283 33554482 1 1089290023 0 0 1089406862 '
            '"raortega" "8nm" "" "" "" "lxplus015" "prog/step3c" "" "/afs/cern.ch/user/r/raortega/log/bstep3c-362.txt" '
            '"/afs/cern.ch/user/r/raortega/log/berr-step3c-362.txt""1089290023.699195" 0 1 "tbed0079" 64 3.3 "" '
            '"/afs/cern.ch/user/r/raortega/prog/step3c/startEachset.pl362 7 8" 277.210000 17.280000 0 0 -1 0 0 927804'
            ' 87722 0 0 0 -1 0 0 0 0 0 -1 "" "default" 0 1 "" "" 0 310424 339112 "" "" ""'
        )

        self.assertRaises(IndexError, self.parser.parse, line)

    def test_invalid_version(self):
        # unsupported version
        line = (
            '"JOB_FINISH" "10.1" 1089407406 699195 283 33554482 1 1089290023 0 0 1089406862 '
            '"raortega" "8nm" "" "" "" "lxplus015" "prog/step3c" "" "/afs/cern.ch/user/r/raortega/log/bstep3c-362.txt" '
            '"/afs/cern.ch/user/r/raortega/log/berr-step3c-362.txt" "1089290023.699195" 0 1 "tbed0079" 64 3.3 "" '
            '"/afs/cern.ch/user/r/raortega/prog/step3c/startEachset.pl362 7 8" 277.210000 17.280000 0 0 -1 0 0 927804'
            ' 87722 0 0 0 -1 0 0 0 0 0 -1 "" "default" 0 1 "" "" 0 310424 339112 "" "" ""'
        )

        self.assertRaises(KeyError, self.parser.parse, line)
Пример #5
0
class ParserLSFTest(TestCase):
    '''
    Test case for LSF parser
    '''
    
    def setUp(self):
        self.parser = LSFParser('testSite', 'testHost', True)
        

    def test_parse(self):
        line1 = ('"JOB_FINISH" "5.1" 1089407406 699195 283 33554482 1 1089290023 0 0 1089406862 '
                '"raortega" "8nm" "" "" "" "lxplus015" "prog/step3c" "" "/afs/cern.ch/user/r/raortega/log/bstep3c-362.txt" '
                '"/afs/cern.ch/user/r/raortega/log/berr-step3c-362.txt" "1089290023.699195" 0 1 "tbed0079" 64 3.3 "" '
                '"/afs/cern.ch/user/r/raortega/prog/step3c/startEachset.pl 362 7 8" 277.210000 17.280000 0 0 -1 0 0 927804'
                ' 87722 0 0 0 -1 0 0 0 0 0 -1 "" "default" 0 1 "" "" 0 310424 339112 "" "" ""')
        
        line1_values = {"JobName": "699195", 
                        "LocalUserID":"raortega", 
                        "LocalUserGroup": None, 
                        "WallDuration":544, 
                        "CpuDuration": 294,
                        "StartTime": datetime.datetime.utcfromtimestamp(1089406862),
                        "StopTime": datetime.datetime.utcfromtimestamp(1089407406),
                        "MemoryReal":310424,
                        "MemoryVirtual": 339112,
                        "NodeCount": 1,
                        "Processors": 1
                        }
        
        # MPI information
        line2 = ('"JOB_FINISH" "7.06" 1302184020 436491 10001 33816691 16 1302184001 0 0 1302184004 "viglen" "normal" '
                 '" span[ptile=8] cu[maxcus=1]" "" "" "comp000" "benchmarks/IMB_3.0/bin/bsub" "" "IMB.%J.%I.out" '
                 '"IMB.%J.%I.err" "1302184001.436491" 0 16 "comp042" "comp042" "comp042" "comp042" "comp042" "comp042" '
                 '"comp042" "comp042" "comp043" "comp043" "comp043" "comp043" "comp043" "comp043" "comp043" "comp043" 32'
                 ' 60.0 "imb_sr[1-90]" "#!/bin/bash; #BSUB -n 16;#BSUB -R \'span[ptile=8]\';#BSUB -R \'cu[maxcus=1]\';#BSUB '
                 '-x;#BSUB -J imb_sr[1-90];####BSUB -m ""comp000 comp001"";#BSUB -o IMB.%J.%I.out;#BSUB -e IMB.%J.%I.err; '
                 '# Output some env vars;echo ""LSB_HOSTS: $LSB_HOSTS"";echo ""LSB_MCPU_HOSTS: $LSB_MCPU_HOSTS"";echo '
                 '""LSB_DJOB_HOSTFILE: $LSB_DJOB_HOSTFILE"";echo ""LSB_DJOB_HOSTFILE (cat):""; cat $LSB_DJOB_HOSTFILE; '
                 'HOSTLIST=`cat $LSB_DJOB_HOSTFILE | uniq | tr ""\n"" "" ""`; . /etc/profile; module add MPI/PMPI;cd '
                 '/home/test/viglen/benchmarks/IMB_3.0/bin/ ;#mpirun -np $LSB_DJOB_NUMPROC ./imb-pmpi -msglen ./IMB_msglen '
                 '-map 2x8 -npmin 16 sendrecv;#mpirun -d -prot -np $LSB_DJOB_NUMPROC -hostlist ""comp000 comp001"" '
                 './imb-pmpi -msglen ./IMB_msglen -map 2x8 -npmin 16 sendrecv;mpirun -d -prot -np $LSB_DJOB_NUMPROC '
                 '-hostlist ""$HOSTLIST"" ./imb-pmpi -msglen ./IMB_msglen -map 2x8 -npmin 16 sendrecv" 0.472928 1.112830 '
                 '0 0 -1 0 0 149746 0 0 0 0 -1 0 0 0 1907 1746 -1 "" "default" 65280 16 "" "" 23 2 25 "" "" "" "" 0 "" 0 ""'
                 ' -1 "/viglen" "" "default" "" -1 "" "" 6160  "" 1302184004 "" "" 0')

        line2_values = {"JobName": "436491", 
                        "LocalUserID":"viglen", 
                        "LocalUserGroup": None, 
                        "WallDuration":16, 
                        "CpuDuration": 2,
                        "StartTime": datetime.datetime.utcfromtimestamp(1302184004),
                        "StopTime": datetime.datetime.utcfromtimestamp(1302184020),
                        "MemoryReal":2,
                        "MemoryVirtual": 25,
                        "NodeCount": 2,
                        "Processors": 16
                        }
        cases = {}
        cases[line1] = line1_values
        cases[line2] = line2_values
        
        for line in cases.keys():
        
            record = self.parser.parse(line)
            cont = record._record_content
        
            self.assertTrue(cont.has_key("Site"))
            self.assertTrue(cont.has_key("JobName"))
            self.assertTrue(cont.has_key("LocalUserID"))
            self.assertTrue(cont.has_key("LocalUserGroup"))
            self.assertTrue(cont.has_key("WallDuration"))
            self.assertTrue(cont.has_key("CpuDuration"))
            self.assertTrue(cont.has_key("StartTime"))
            self.assertTrue(cont.has_key("StopTime"))
            self.assertTrue(cont.has_key("MemoryReal"))
            self.assertTrue(cont.has_key("MemoryVirtual"))
        
            for key in cases[line].keys():
                self.assertEqual(cont[key], cases[line][key], "%s != %s for key %s" % (cont[key], cases[line][key], key))
        
        
    def test_invalid_expr(self):
        # two fields are not separated by space
        line = ('"JOB_FINISH" "5.1" 1089407406 699195 283 33554482 1 1089290023 0 0 1089406862 '
                '"raortega" "8nm" "" "" "" "lxplus015" "prog/step3c" "" "/afs/cern.ch/user/r/raortega/log/bstep3c-362.txt" '
                '"/afs/cern.ch/user/r/raortega/log/berr-step3c-362.txt""1089290023.699195" 0 1 "tbed0079" 64 3.3 "" '
                '"/afs/cern.ch/user/r/raortega/prog/step3c/startEachset.pl362 7 8" 277.210000 17.280000 0 0 -1 0 0 927804'
                ' 87722 0 0 0 -1 0 0 0 0 0 -1 "" "default" 0 1 "" "" 0 310424 339112 "" "" ""')
        
        self.assertRaises(IndexError, self.parser.parse, line)
    
    def test_invalid_version(self):
        # unsupported version
        line = ('"JOB_FINISH" "9.1" 1089407406 699195 283 33554482 1 1089290023 0 0 1089406862 '
                '"raortega" "8nm" "" "" "" "lxplus015" "prog/step3c" "" "/afs/cern.ch/user/r/raortega/log/bstep3c-362.txt" '
                '"/afs/cern.ch/user/r/raortega/log/berr-step3c-362.txt" "1089290023.699195" 0 1 "tbed0079" 64 3.3 "" '
                '"/afs/cern.ch/user/r/raortega/prog/step3c/startEachset.pl362 7 8" 277.210000 17.280000 0 0 -1 0 0 927804'
                ' 87722 0 0 0 -1 0 0 0 0 0 -1 "" "default" 0 1 "" "" 0 310424 339112 "" "" ""')
        
        self.assertRaises(KeyError, self.parser.parse, line)