Esempio n. 1
0
    def test_cluster_from_options(self):
        """Test cluster from options with ugent/vsc regexp"""

        self.assertEqual(
            cluster_from_options(
                [
                    ("l", "x"),
                    ("q", "*****@*****.**"),
                    ("m", "more"),
                    ("q", "*****@*****.**"),
                    ("N", None),
                ],
                MASTER_REGEXP,
            ),
            "whatever",
            msg="found expected cluster based on last queue option",
        )

        os.environ["PBS_DEFAULT"] = "master10.mycluster.gent.vsc"
        self.assertEqual(
            cluster_from_options(
                [("l", "x"), ("q", "long"), ("m", "more"), ("q", "short"), ("N", None)], MASTER_REGEXP
            ),
            "mycluster",
            msg="found expected cluster based on PBS_DEFAULT environment variable %s" % os.environ["PBS_DEFAULT"],
        )

        self.assertEqual(get_warnings(), [], msg="no cluster warnings")

        os.environ["PBS_DEFAULT"] = "master10.notgent.vsc"
        self.assertEqual(
            cluster_from_options(
                [("l", "x"), ("q", "long"), ("m", "more"), ("q", "short"), ("N", None)], MASTER_REGEXP
            ),
            DEFAULT_SERVER_CLUSTER,
            msg="no cluster found, fallback to DEFAULT_SERVER_CLUSTER %s" % DEFAULT_SERVER_CLUSTER,
        )

        self.assertEqual(
            get_warnings(),
            [
                "Unable to determine clustername, using default delcatty (queue short, PBS_DEFAULT %s)"
                % os.environ["PBS_DEFAULT"]
            ],
            msg="no cluster found, warnings generated %s" % (get_warnings()),
        )

        del os.environ["PBS_DEFAULT"]
Esempio n. 2
0
def main(arguments=None):
    """Main function"""

    if arguments is None:
        arguments = sys.argv
    
    sf = SubmitFilter(arguments, sys.stdin.readline)
    sf.parse_header()

    header = make_new_header(sf)

    # flush it so it doesn't get mixed with stderr
    sys.stdout.flush()
    sys.stderr.flush()

    # prebody is not stripped of the newline
    sys.stdout.write("\n".join(header+[sf.prebody]))
    for line in sf.stdin:
        sys.stdout.write(line)
    
    # print all generated warnings
    # flush it so it doesn't get mixed with stderr
    sys.stdout.flush()
    for warn in ["%s\n" % w for w in get_warnings()]:
        sys.stderr.write(warn)
    sys.stderr.flush()

    sys.exit(0)
Esempio n. 3
0
    def test_nodesfilter(self):
        """Test parse_resources_nodes"""
        for testdata in RESOURCE_NODES:
            orig = testdata.pop("orig")
            resources = {}
            nodes = parse_resources_nodes(orig, "gengar", resources)
            self.assertEqual(
                "nodes=" + resources["nodes"],
                nodes,
                msg="nodes returned is equal to nodes=resources[nodes] (nodes=%s == %s)" % (resources["nodes"], nodes),
            )
            self.assertEqual(
                resources,
                testdata,
                msg="generated resources equal to expected (expected %s generated %s; orig %s)"
                % (testdata, resources, orig),
            )

        # up to now, 2 warnings
        warnings = get_warnings()
        self.assertEqual(
            warnings,
            [
                "Warning: unknown ppn (whatever) detected, using ppn=1",
                "Warning: unknown ppn (woohoo) detected, using ppn=1",
            ],
            msg="expected warnings after processing examples %s" % (warnings),
        )

        reset_warnings()
Esempio n. 4
0
    def test_nodesfilter(self):
        """Test parse_resources_nodes"""
        for testdata in RESOURCE_NODES:
            orig = testdata.pop('orig')
            resources = {}
            nodes = parse_resources_nodes(orig, 'gengar', resources)
            self.assertEqual(
                'nodes=' + resources['nodes'],
                nodes,
                msg=
                'nodes returned is equal to nodes=resources[nodes] (nodes=%s == %s)'
                % (resources['nodes'], nodes))
            self.assertEqual(
                resources,
                testdata,
                msg=
                'generated resources equal to expected (expected %s generated %s; orig %s)'
                % (testdata, resources, orig))

        # up to now, 2 warnings
        warnings = get_warnings()
        self.assertEqual(warnings, [
            'Warning: unknown ppn (whatever) detected, using ppn=1',
            'Warning: unknown ppn (woohoo) detected, using ppn=1',
        ],
                         msg="expected warnings after processing examples %s" %
                         (warnings))

        reset_warnings()
Esempio n. 5
0
    def test_make_new_header_pmem_limits(self, mock_cluster_overhead, mock_clusterdata):
        sf = SubmitFilter(
            [],
            [x + "\n" for x in SCRIPTS[6].split("\n")]
        )

        mock_clusterdata.return_value = {
            'TOTMEM': 4096 << 20,
            'PHYSMEM': 3072 << 20,
            'NP': 8,
            'NP_LCD': 2,
        }
        mock_cluster_overhead.return_value = 0

        sf.parse_header()
        header = submitfilter.make_new_header(sf)
        self.assertEqual(header, [
            '#!/bin/bash',
            '#PBS -l nodes=1:ppn=4',
            '#PBS -l pmem=1g',
            '#PBS -m n'
            '',
            '',
        ], msg='header with existing mem set')
        self.assertEqual(get_warnings(), [
            "Unable to determine clustername, using default delcatty (no PBS_DEFAULT)",
            "Warning, requested %sb pmem per node, this is more than the available pmem (%sb), this job will never start." % (1 << 30, (3072 << 20) / 8 )
        ])
Esempio n. 6
0
    def test_cluster_from_options(self):
        """Test cluster from options with ugent/vsc regexp"""

        self.assertEqual(cluster_from_options([
            ('l', 'x'),
            ('q', '*****@*****.**'),
            ('m', 'more'),
            ('q', '*****@*****.**'),
            ('N', None)
        ], MASTER_REGEXP), 'whatever',
                         msg="found expected cluster based on last queue option")

        os.environ['PBS_DEFAULT'] = "master10.mycluster.gent.vsc"
        self.assertEqual(cluster_from_options([
            ('l', 'x'),
            ('q', 'long'),
            ('m', 'more'),
            ('q', 'short'),
            ('N', None)
        ], MASTER_REGEXP), 'mycluster',
                         msg="found expected cluster based on PBS_DEFAULT environment variable %s" % os.environ['PBS_DEFAULT'])


        self.assertEqual(get_warnings(), [], msg='no cluster warnings')

        os.environ['PBS_DEFAULT'] = "master10.notgent.vsc"
        self.assertEqual(cluster_from_options([
            ('l', 'x'),
            ('q', 'long'),
            ('m', 'more'),
            ('q', 'short'),
            ('N', None)
        ], MASTER_REGEXP), DEFAULT_SERVER_CLUSTER,
                         msg="no cluster found, fallback to DEFAULT_SERVER_CLUSTER %s" % DEFAULT_SERVER_CLUSTER)

        self.assertEqual(get_warnings(), [
            'Unable to determine clustername, using default delcatty (queue short, PBS_DEFAULT %s)' % os.environ['PBS_DEFAULT'],
        ], msg='no cluster found, warnings generated %s' %(get_warnings()))

        del os.environ['PBS_DEFAULT']
Esempio n. 7
0
    def test_cluster_from_options(self):
        """Test cluster from options with ugent/vsc regexp"""

        self.assertEqual(
            cluster_from_options([('l', 'x'),
                                  ('q', '*****@*****.**'),
                                  ('m', 'more'),
                                  ('q', '*****@*****.**'),
                                  ('N', None)], MASTER_REGEXP),
            'whatever',
            msg="found expected cluster based on last queue option")

        os.environ['PBS_DEFAULT'] = "master10.mycluster.gent.vsc"
        self.assertEqual(
            cluster_from_options([('l', 'x'), ('q', 'long'), ('m', 'more'),
                                  ('q', 'short'), ('N', None)], MASTER_REGEXP),
            'mycluster',
            msg=
            "found expected cluster based on PBS_DEFAULT environment variable %s"
            % os.environ['PBS_DEFAULT'])

        self.assertEqual(get_warnings(), [], msg='no cluster warnings')

        os.environ['PBS_DEFAULT'] = "master10.notgent.vsc"
        self.assertEqual(
            cluster_from_options([('l', 'x'), ('q', 'long'), ('m', 'more'),
                                  ('q', 'short'), ('N', None)], MASTER_REGEXP),
            DEFAULT_SERVER_CLUSTER,
            msg="no cluster found, fallback to DEFAULT_SERVER_CLUSTER %s" %
            DEFAULT_SERVER_CLUSTER)

        self.assertEqual(get_warnings(), [
            'Unable to determine clustername, using default delcatty (queue short, PBS_DEFAULT %s)'
            % os.environ['PBS_DEFAULT'],
        ],
                         msg='no cluster found, warnings generated %s' %
                         (get_warnings()))

        del os.environ['PBS_DEFAULT']
Esempio n. 8
0
    def test_nodesfilter(self):
        """Test parse_resources_nodes"""
        for testdata in RESOURCE_NODES:
            orig = testdata.pop('orig')
            resources = {}
            nodes = parse_resources_nodes(orig, 'gengar', resources)
            self.assertEqual('nodes=' + resources['nodes'], nodes,
                             msg='nodes returned is equal to nodes=resources[nodes] (nodes=%s == %s)' % (resources['nodes'], nodes))
            self.assertEqual(resources, testdata, msg='generated resources equal to expected (expected %s generated %s; orig %s)' %
                             (testdata, resources, orig))

        # up to now, 2 warnings
        warnings = get_warnings()
        self.assertEqual(warnings,
                         ['Warning: unknown ppn (whatever) detected, using ppn=1',
                          'Warning: unknown ppn (woohoo) detected, using ppn=1',
                         ], msg="expected warnings after processing examples %s" % (warnings))

        reset_warnings()
Esempio n. 9
0
    def test_make_new_header_warn(self):
        """
        Test make_new_header warnings
            ideal ppn
            vmem too high
        """
        reset_warnings()

        sf = SubmitFilter(
            [],
            [x + "\n" for x in SCRIPTS[3].split("\n")]
        )
        sf.parse_header()

        header = submitfilter.make_new_header(sf)
        self.assertEqual(header, sf.header, msg='unmodified header')
        self.assertEqual(get_warnings(), [
            'The chosen ppn 4 is not considered ideal: should use either lower than or multiple of 3',
            'Warning, requested 1099511627776b vmem per node, this is more than the available vmem (86142287872b), this job will never start.',
        ], msg='warnings for ideal ppn and vmem too high')
Esempio n. 10
0
    def test_make_new_header_warn(self):
        """
        Test make_new_header warnings
            ideal ppn
            vmem too high
        """
        reset_warnings()

        sf = SubmitFilter(
            [],
            [x + "\n" for x in SCRIPTS[3].split("\n")]
        )
        sf.parse_header()

        header = submitfilter.make_new_header(sf)
        self.assertEqual(header, sf.header, msg='unmodified header')
        self.assertEqual(get_warnings(), [
            'The chosen ppn 4 is not considered ideal: should use either lower than or multiple of 3',
            'Warning, requested 1099511627776b vmem per node, this is more than the available vmem (86142287872b), this job will never start.',
        ], msg='warnings for ideal ppn and vmem too high')
Esempio n. 11
0
def main(arguments=None):
    """Main function"""

    if arguments is None:
        arguments = sys.argv

    # This error could otherwise result in empty PBS_O_WORKDIR
    try:
        os.getcwd()
    except OSError as e:
        sys.stderr.write("ERROR: Unable to determine current workdir: %s (PWD deleted?)." % e)
        sys.stderr.flush()
        sys.exit(1)

    sf = SubmitFilter(arguments, sys.stdin.readline)
    sf.parse_header()

    header = make_new_header(sf)

    # flush it so it doesn't get mixed with stderr
    sys.stdout.flush()
    sys.stderr.flush()

    # prebody is not stripped of the newline
    sys.stdout.write("\n".join(header+[sf.prebody]))
    for line in sf.stdin:
        sys.stdout.write(line)

    # print all generated warnings
    # flush it so it doesn't get mixed with stderr
    sys.stdout.flush()
    for warning in ["%s\n" % w for w in get_warnings()]:
        sys.stderr.write(warning)
    sys.stderr.flush()

    sys.exit(0)
Esempio n. 12
0
def main(arguments=None):
    """Main function"""

    if arguments is None:
        arguments = sys.argv

    # This error could otherwise result in empty PBS_O_WORKDIR
    try:
        os.getcwd()
    except OSError as e:
        sys.stderr.write("ERROR: Unable to determine current workdir: %s (PWD deleted?)." % e)
        sys.stderr.flush()
        sys.exit(1)

    sf = SubmitFilter(arguments, sys.stdin.readline)
    sf.parse_header()

    header = make_new_header(sf)

    # flush it so it doesn't get mixed with stderr
    sys.stdout.flush()
    sys.stderr.flush()

    # prebody is not stripped of the newline
    sys.stdout.write("\n".join(header+[sf.prebody]))
    for line in sf.stdin:
        sys.stdout.write(line)

    # print all generated warnings
    # flush it so it doesn't get mixed with stderr
    sys.stdout.flush()
    for warning in ["%s\n" % w for w in get_warnings()]:
        sys.stderr.write(warning)
    sys.stderr.flush()

    sys.exit(0)