def test_dprefix(self): """Test dprefix and usage in parseline and make_header""" h = SubmitFilter([], []) self.assertEqual(PBS_DIRECTIVE_PREFIX_DEFAULT, '#PBS', msg='PBS_DIRECTIVE_PREFIX_DEFAULT %s' % PBS_DIRECTIVE_PREFIX_DEFAULT) self.assertEqual( h.dprefix, PBS_DIRECTIVE_PREFIX_DEFAULT, msg='PBS_DIRECTIVE_PREFIX_DEFAULT set as default dprefix') # test parseline with default dprefix self.assertEqual(h.parseline('regular line of text'), None, msg='non-header returns None') self.assertEqual(h.parseline('# sldjldkfjkg'), [], msg='regular header returns empty list') self.assertEqual(h.parseline(' '), [], msg='whitespace returns empty list') res = h.parseline('#PBS -x y=z -a b=c,d=e,f,g # some comment') self.assertEqual( h.parseline('#PBS -x y=z -a b=c,d=e,f,g # some comment'), [('x', 'y=z'), ('a', 'b=c,d=e,f,g')], msg='PBS header with 2 options and comment (%s)' % (res, )) self.assertEqual(h.make_header('-a', 'b'), '#PBS -a b', 'Generate correct header') # drpefix from enviornment os.environ['PBS_DPREFIX'] = '#ENVDPREFIX' h1 = SubmitFilter([], []) self.assertEqual(h1.dprefix, '#ENVDPREFIX', msg='dprefix from environment wins') # different dprefix from commandline h2 = SubmitFilter(['-C', '#SOMETHINGELSE'], []) self.assertEqual(h2.dprefix, '#SOMETHINGELSE', msg='cmdline dprefix wins') # very different dprefix self.assertEqual(h2.parseline('#PBS -x y=z -a b'), [], 'new dprefix, this is now just a comment') self.assertEqual(h2.parseline('#SOMETHINGELSE -x y=z -a b'), [('x', 'y=z'), ('a', 'b')], 'new dprefix, this is now just a comment') self.assertEqual(h2.make_header('-a', 'b'), '#SOMETHINGELSE -a b', 'Generate correct header with modified dprefix') del os.environ['PBS_DPREFIX']
def test_make_new_header_add_missing(self): """ Test make_new_header add missing mail / unless present add pvmem unless defined VSC_NODE_PARTITION """ partname = 'mypartition' os.environ['VSC_NODE_PARTITION'] = partname sf = SubmitFilter( [], [x + "\n" for x in SCRIPTS[1].split("\n")] ) sf.parse_header() self.assertEqual(sf.header, ['#!/bin/bash'], msg='minimal header from minimal script') header = submitfilter.make_new_header(sf) self.assertEqual(header, [ '#!/bin/bash', '# No mail specified - added by submitfilter', '#PBS -m n', '# No pmem or vmem limit specified - added by submitfilter (server found: delcatty)', '#PBS -l vmem=4720302336', '# Adding PARTITION as specified in VSC_NODE_PARTITION', '#PBS -W x=PARTITION:%s' % partname, ], msg='added missing defaults and pratiton information to header') del os.environ['VSC_NODE_PARTITION']
def test_gather_state(self): """Test gather_state""" h = SubmitFilter(['-q', 'verylong'], [x + "\n" for x in SCRIPTS[0].split("\n")]) h.parse_header() os.environ['PBS_DEFAULT'] = "master15.delcatty.gent.vsc" overhead = 2842984448 allpmem = 67630407680 - overhead state, newopts = h.gather_state(MASTER_REGEXP) self.assertTrue( 'l' in state, msg='state retruned by gather_state always contains key "l"') self.assertEqual(state, { 'e': 'error_testrun.txt', 'm': 'bea', 'l': { '_nrcores': 80, '_ppn': 16, 'vmem': '500mb', '_vmem': 500 * 2**20, '_nrnodes': 5, 'nodes': '5:ppn=16', 'walltime': '11:25:00', 'pmem': '%s' % (allpmem / 2), '_pmem': allpmem / 2, }, 'o': 'output_testrun.txt', 'N': 'testrun', 'q': 'verylong', '_cluster': 'delcatty' }, msg="expected state %s" % state) self.assertEqual(newopts, [ 'testrun', 'output_testrun.txt', 'nodes=5:ppn=16,pmem=%s' % (allpmem / 2), 'error_testrun.txt', 'walltime=11:25:00', 'vmem=500mb', 'bea', 'short', 'verylong', ], msg="expected newopts %s" % newopts) del os.environ['PBS_DEFAULT']
def test_parse(self): h = SubmitFilter(['-q', 'verylong'], [x + "\n" for x in SCRIPTS[0].split("\n")]) # stdin is an iterator self.assertTrue(hasattr(h.stdin, 'next'), "stdin is an iterator") h.parse_header() self.assertEqual(h.header, [ '#!/bin/sh', '#', '#', '#PBS -N testrun', '#PBS -o output_testrun.txt -l nodes=5:ppn=all,pmem=half', '#PBS -e error_testrun.txt', '#PBS -l walltime=11:25:00', '#PBS -l vmem=500mb', '#PBS -m bea', '#PBS -q short', '#', ], msg="Found header %s" % h.header) self.assertEqual(h.prebody, "cd $VSC_HOME\n", msg="found prebody '%s'" % h.prebody) self.assertEqual(h.dprefix, '#PBS', msg="found expected headers dprefix %s" % h.dprefix) self.assertEqual( h.stdin.next(), "##logs to stderr by default, redirect this to stdout\n", msg="stdin at expected position") self.assertEqual(h.allopts, [ ('N', 'testrun'), ('o', 'output_testrun.txt'), ('l', 'nodes=5:ppn=all,pmem=half'), ('e', 'error_testrun.txt'), ('l', 'walltime=11:25:00'), ('l', 'vmem=500mb'), ('m', 'bea'), ('q', 'short'), ('q', 'verylong'), ], msg="found alloptions in order %s" % h.allopts) self.assertEqual(h.occur, [3, 4, 4, 5, 6, 7, 8, 9, None], msg="expected ordered occurence of options %s" % h.occur)
def test_make_new_header_ignore_indentation(self): sf = SubmitFilter( [], [x + "\n" for x in SCRIPTS[5].split("\n")] ) sf.parse_header() header = submitfilter.make_new_header(sf) self.assertEqual(header, [ '#!/bin/bash', '#PBS -l nodes=1:ppn=4', '#PBS -l vmem=1g', '#PBS -m n', '', ], msg='header with an indented line')
def test_make_new_header_with_existing_mem(self): sf = SubmitFilter( [], [x + "\n" for x in SCRIPTS[4].split("\n")] ) sf.parse_header() header = submitfilter.make_new_header(sf) self.assertEqual(header, [ '#!/bin/bash', '#PBS -l nodes=1:ppn=4', '#PBS -l mem=10g', '#PBS -m n' '', '', ], msg='header with existing mem set')
def test_make_new_header(self): """Test make_new_header resource replacement""" sf = SubmitFilter( [], [x + "\n" for x in SCRIPTS[2].split("\n")] ) sf.parse_header() header = submitfilter.make_new_header(sf) self.assertEqual(header, [ '#!/bin/bash', '#PBS -l nodes=3:ppn=8', '#PBS -l vmem=75524837376', '#PBS -l pmem=32393711616', '#PBS -m n', ], msg='modified header with resources replaced')
def test_make_new_header_basic(self): """Basic test for make_new_header""" sf = SubmitFilter( ['-q', 'verylong'], [x + "\n" for x in SCRIPTS[0].split("\n")] ) sf.parse_header() old_header = sf.header[:] header = submitfilter.make_new_header(sf) self.assertEqual(old_header, sf.header, msg='make_new_header leaves original unmodified') modresourcesidx = 4 self.assertEqual(header.pop(modresourcesidx), '#PBS -o output_testrun.txt -l nodes=5:ppn=16,pmem=32393711616', msg='replace the resource header as expected') sf.header.pop(modresourcesidx) self.assertEqual(header, sf.header, msg='all other header lines unmodified')
def test_make_new_header_warn(self): """ Test make_new_header warnings ideal ppn vmem too high """ reset_warnings() sf = SubmitFilter( [], [x + "\n" for x in SCRIPTS[3].split("\n")] ) sf.parse_header() header = submitfilter.make_new_header(sf) self.assertEqual(header, sf.header, msg='unmodified header') self.assertEqual(get_warnings(), [ 'The chosen ppn 4 is not considered ideal: should use either lower than or multiple of 3', 'Warning, requested 1099511627776b vmem per node, this is more than the available vmem (86142287872b), this job will never start.', ], msg='warnings for ideal ppn and vmem too high')
def main(arguments=None): """Main function""" if arguments is None: arguments = sys.argv # This error could otherwise result in empty PBS_O_WORKDIR try: os.getcwd() except OSError as e: sys.stderr.write("ERROR: Unable to determine current workdir: %s (PWD deleted?)." % e) sys.stderr.flush() sys.exit(1) sf = SubmitFilter(arguments, sys.stdin.readline) sf.parse_header() header = make_new_header(sf) # flush it so it doesn't get mixed with stderr sys.stdout.flush() sys.stderr.flush() # prebody is not stripped of the newline sys.stdout.write("\n".join(header+[sf.prebody])) for line in sf.stdin: sys.stdout.write(line) # print all generated warnings # flush it so it doesn't get mixed with stderr sys.stdout.flush() for warning in ["%s\n" % w for w in get_warnings()]: sys.stderr.write(warning) sys.stderr.flush() sys.exit(0)