示例#1
0
    def test_make_new_header_add_missing(self):
        """
        Test make_new_header
          add missing mail / unless present
          add pvmem unless defined
          VSC_NODE_PARTITION
        """

        partname = 'mypartition'
        os.environ['VSC_NODE_PARTITION'] = partname
        sf = SubmitFilter(
            [],
            [x + "\n" for x in SCRIPTS[1].split("\n")]
        )
        sf.parse_header()

        self.assertEqual(sf.header, ['#!/bin/bash'], msg='minimal header from minimal script')
        header = submitfilter.make_new_header(sf)
        self.assertEqual(header, [
            '#!/bin/bash',
            '# No mail specified - added by submitfilter',
            '#PBS -m n',
            '# No pmem or vmem limit specified - added by submitfilter (server found: delcatty)',
            '#PBS -l vmem=4720302336',
            '# Adding PARTITION as specified in VSC_NODE_PARTITION',
            '#PBS -W x=PARTITION:%s' % partname,
        ], msg='added missing defaults and pratiton information to header')

        del os.environ['VSC_NODE_PARTITION']
示例#2
0
    def test_make_new_header_add_missing(self):
        """
        Test make_new_header
          add missing mail / unless present
          add pvmem unless defined
          VSC_NODE_PARTITION
        """

        partname = "mypartition"
        os.environ["VSC_NODE_PARTITION"] = partname
        sf = SubmitFilter([], [x + "\n" for x in SCRIPTS[1].split("\n")])
        sf.parse_header()

        self.assertEqual(sf.header, ["#!/bin/bash"], msg="minimal header from minimal script")
        header = submitfilter.make_new_header(sf)
        self.assertEqual(
            header,
            [
                "#!/bin/bash",
                "# No mail specified - added by submitfilter",
                "#PBS -m n",
                "# No pmem or vmem limit specified - added by submitfilter (server found: delcatty)",
                "#PBS -l vmem=4720302336",
                "# Adding PARTITION as specified in VSC_NODE_PARTITION",
                "#PBS -W x=PARTITION:%s" % partname,
            ],
            msg="added missing defaults and pratiton information to header",
        )

        del os.environ["VSC_NODE_PARTITION"]
    def test_make_new_header_pmem_limits(self, mock_cluster_overhead, mock_clusterdata):
        sf = SubmitFilter(
            [],
            [x + "\n" for x in SCRIPTS[6].split("\n")]
        )

        mock_clusterdata.return_value = {
            'TOTMEM': 4096 << 20,
            'PHYSMEM': 3072 << 20,
            'NP': 8,
            'NP_LCD': 2,
        }
        mock_cluster_overhead.return_value = 0

        sf.parse_header()
        header = submitfilter.make_new_header(sf)
        self.assertEqual(header, [
            '#!/bin/bash',
            '#PBS -l nodes=1:ppn=4',
            '#PBS -l pmem=1g',
            '#PBS -m n'
            '',
            '',
        ], msg='header with existing mem set')
        self.assertEqual(get_warnings(), [
            "Unable to determine clustername, using default delcatty (no PBS_DEFAULT)",
            "Warning, requested %sb pmem per node, this is more than the available pmem (%sb), this job will never start." % (1 << 30, (3072 << 20) / 8 )
        ])
示例#4
0
 def test_make_new_header_ignore_indentation(self):
     sf = SubmitFilter([], [x + "\n" for x in SCRIPTS[5].split("\n")])
     sf.parse_header()
     header = submitfilter.make_new_header(sf)
     self.assertEqual(
         header,
         ["#!/bin/bash", "#PBS -l nodes=1:ppn=4", "#PBS -l vmem=1g", "#PBS -m n", ""],
         msg="header with an indented line",
     )
示例#5
0
 def test_make_new_header_with_existing_mem(self):
     sf = SubmitFilter([], [x + "\n" for x in SCRIPTS[4].split("\n")])
     sf.parse_header()
     header = submitfilter.make_new_header(sf)
     self.assertEqual(
         header,
         ["#!/bin/bash", "#PBS -l nodes=1:ppn=4", "#PBS -l mem=10g", "#PBS -m n" "", ""],
         msg="header with existing mem set",
     )
示例#6
0
 def test_make_new_header_ignore_indentation(self):
     sf = SubmitFilter(
         [],
         [x + "\n" for x in SCRIPTS[5].split("\n")]
     )
     sf.parse_header()
     header = submitfilter.make_new_header(sf)
     self.assertEqual(header, [
         '#!/bin/bash',
         '#PBS -l nodes=1:ppn=4',
         '#PBS -l vmem=1g',
         '#PBS -m n',
         '',
     ], msg='header with an indented line')
示例#7
0
 def test_make_new_header_with_existing_mem(self):
     sf = SubmitFilter(
         [],
         [x + "\n" for x in SCRIPTS[4].split("\n")]
     )
     sf.parse_header()
     header = submitfilter.make_new_header(sf)
     self.assertEqual(header, [
         '#!/bin/bash',
         '#PBS -l nodes=1:ppn=4',
         '#PBS -l mem=10g',
         '#PBS -m n'
         '',
         '',
     ], msg='header with existing mem set')
 def test_make_new_header_with_existing_mem(self):
     sf = SubmitFilter(
         [],
         [x + "\n" for x in SCRIPTS[4].split("\n")]
     )
     sf.parse_header()
     header = submitfilter.make_new_header(sf)
     self.assertEqual(header, [
         '#!/bin/bash',
         '#PBS -l nodes=1:ppn=4',
         '#PBS -l mem=4g',
         '#PBS -m n'
         '',
         '',
     ], msg='header with existing mem set')
示例#9
0
    def test_make_new_header(self):
        """Test make_new_header resource replacement"""
        sf = SubmitFilter(
            [],
            [x + "\n" for x in SCRIPTS[2].split("\n")]
        )
        sf.parse_header()

        header = submitfilter.make_new_header(sf)
        self.assertEqual(header, [
            '#!/bin/bash',
            '#PBS -l nodes=3:ppn=8',
            '#PBS -l vmem=75524837376',
            '#PBS -l pmem=32393711616',
            '#PBS -m n',
        ], msg='modified header with resources replaced')
示例#10
0
    def test_make_new_header(self):
        """Test make_new_header resource replacement"""
        sf = SubmitFilter(
            [],
            [x + "\n" for x in SCRIPTS[2].split("\n")]
        )
        sf.parse_header()

        header = submitfilter.make_new_header(sf)
        self.assertEqual(header, [
            '#!/bin/bash',
            '#PBS -l nodes=3:ppn=8',
            '#PBS -l vmem=75524837376',
            '#PBS -l pmem=32393711616',
            '#PBS -m n',
        ], msg='modified header with resources replaced')
示例#11
0
    def test_make_new_header_basic(self):
        """Basic test for make_new_header"""
        sf = SubmitFilter(["-q", "verylong"], [x + "\n" for x in SCRIPTS[0].split("\n")])
        sf.parse_header()

        old_header = sf.header[:]
        header = submitfilter.make_new_header(sf)
        self.assertEqual(old_header, sf.header, msg="make_new_header leaves original unmodified")

        modresourcesidx = 4

        self.assertEqual(
            header.pop(modresourcesidx),
            "#PBS -o output_testrun.txt -l nodes=5:ppn=16,pmem=32393711616",
            msg="replace the resource header as expected",
        )

        sf.header.pop(modresourcesidx)
        self.assertEqual(header, sf.header, msg="all other header lines unmodified")
示例#12
0
    def test_make_new_header_basic(self):
        """Basic test for make_new_header"""
        sf = SubmitFilter(
            ['-q', 'verylong'],
            [x + "\n" for x in SCRIPTS[0].split("\n")]
        )
        sf.parse_header()

        old_header = sf.header[:]
        header = submitfilter.make_new_header(sf)
        self.assertEqual(old_header, sf.header, msg='make_new_header leaves original unmodified')

        modresourcesidx = 4

        self.assertEqual(header.pop(modresourcesidx),
                         '#PBS -o output_testrun.txt -l nodes=5:ppn=16,pmem=32393711616',
                         msg='replace the resource header as expected')

        sf.header.pop(modresourcesidx)
        self.assertEqual(header, sf.header, msg='all other header lines unmodified')
示例#13
0
    def test_make_new_header_warn(self):
        """
        Test make_new_header warnings
            ideal ppn
            vmem too high
        """
        reset_warnings()

        sf = SubmitFilter(
            [],
            [x + "\n" for x in SCRIPTS[3].split("\n")]
        )
        sf.parse_header()

        header = submitfilter.make_new_header(sf)
        self.assertEqual(header, sf.header, msg='unmodified header')
        self.assertEqual(get_warnings(), [
            'The chosen ppn 4 is not considered ideal: should use either lower than or multiple of 3',
            'Warning, requested 1099511627776b vmem per node, this is more than the available vmem (86142287872b), this job will never start.',
        ], msg='warnings for ideal ppn and vmem too high')
示例#14
0
    def test_make_new_header_warn(self):
        """
        Test make_new_header warnings
            ideal ppn
            vmem too high
        """
        reset_warnings()

        sf = SubmitFilter(
            [],
            [x + "\n" for x in SCRIPTS[3].split("\n")]
        )
        sf.parse_header()

        header = submitfilter.make_new_header(sf)
        self.assertEqual(header, sf.header, msg='unmodified header')
        self.assertEqual(get_warnings(), [
            'The chosen ppn 4 is not considered ideal: should use either lower than or multiple of 3',
            'Warning, requested 1099511627776b vmem per node, this is more than the available vmem (86142287872b), this job will never start.',
        ], msg='warnings for ideal ppn and vmem too high')
示例#15
0
    def test_make_new_header_add_missing(self):
        """
        Test make_new_header
          add missing mail / unless present
          add pvmem unless defined
          VSC_NODE_PARTITION
        """

        partname = 'mypartition'
        os.environ[submitfilter.ENV_NODE_PARTITION] = partname

        reserv = 'magicreserv'
        os.environ[submitfilter.ENV_RESERVATION] = reserv

        sf = SubmitFilter(
            [],
            [x + "\n" for x in SCRIPTS[1].split("\n")]
        )
        sf.parse_header()

        self.assertEqual(sf.header, ['#!/bin/bash'], msg='minimal header from minimal script')
        header = submitfilter.make_new_header(sf)
        self.assertEqual(header, [
            '#!/bin/bash',
            '# No mail specified - added by submitfilter',
            '#PBS -m n',
            '# No pmem or vmem limit specified - added by submitfilter (server found: delcatty)',
            '#PBS -l vmem=4720302336',
            '# Adding PARTITION as specified in VSC_NODE_PARTITION',
            '#PBS -W x=PARTITION:%s' % partname,
            '# Adding reservation as specified in VSC_RESERVATION',
            '#PBS -W x=FLAGS:ADVRES:%s' % reserv,
        ], msg='added missing defaults and pratiton information to header')

        del os.environ[submitfilter.ENV_NODE_PARTITION]
        del os.environ[submitfilter.ENV_RESERVATION]