def test_cluster_from_options(self):
"""Test cluster from options with ugent/vsc regexp"""
self.assertEqual(
cluster_from_options(
[
("l", "x"),
("q", "*****@*****.**"),
("m", "more"),
("q", "*****@*****.**"),
("N", None),
],
MASTER_REGEXP,
),
"whatever",
msg="found expected cluster based on last queue option",
)
os.environ["PBS_DEFAULT"] = "master10.mycluster.gent.vsc"
self.assertEqual(
cluster_from_options(
[("l", "x"), ("q", "long"), ("m", "more"), ("q", "short"), ("N", None)], MASTER_REGEXP
),
"mycluster",
msg="found expected cluster based on PBS_DEFAULT environment variable %s" % os.environ["PBS_DEFAULT"],
)
self.assertEqual(get_warnings(), [], msg="no cluster warnings")
os.environ["PBS_DEFAULT"] = "master10.notgent.vsc"
self.assertEqual(
cluster_from_options(
[("l", "x"), ("q", "long"), ("m", "more"), ("q", "short"), ("N", None)], MASTER_REGEXP
),
DEFAULT_SERVER_CLUSTER,
msg="no cluster found, fallback to DEFAULT_SERVER_CLUSTER %s" % DEFAULT_SERVER_CLUSTER,
)
self.assertEqual(
get_warnings(),
[
"Unable to determine clustername, using default delcatty (queue short, PBS_DEFAULT %s)"
% os.environ["PBS_DEFAULT"]
],
msg="no cluster found, warnings generated %s" % (get_warnings()),
)
del os.environ["PBS_DEFAULT"]
def main(arguments=None):
"""Main function"""
if arguments is None:
arguments = sys.argv
sf = SubmitFilter(arguments, sys.stdin.readline)
sf.parse_header()
header = make_new_header(sf)
# flush it so it doesn't get mixed with stderr
sys.stdout.flush()
sys.stderr.flush()
# prebody is not stripped of the newline
sys.stdout.write("\n".join(header+[sf.prebody]))
for line in sf.stdin:
sys.stdout.write(line)
# print all generated warnings
# flush it so it doesn't get mixed with stderr
sys.stdout.flush()
for warn in ["%s\n" % w for w in get_warnings()]:
sys.stderr.write(warn)
sys.stderr.flush()
sys.exit(0)
def test_nodesfilter(self):
"""Test parse_resources_nodes"""
for testdata in RESOURCE_NODES:
orig = testdata.pop("orig")
resources = {}
nodes = parse_resources_nodes(orig, "gengar", resources)
self.assertEqual(
"nodes=" + resources["nodes"],
nodes,
msg="nodes returned is equal to nodes=resources[nodes] (nodes=%s == %s)" % (resources["nodes"], nodes),
)
self.assertEqual(
resources,
testdata,
msg="generated resources equal to expected (expected %s generated %s; orig %s)"
% (testdata, resources, orig),
)
# up to now, 2 warnings
warnings = get_warnings()
self.assertEqual(
warnings,
[
"Warning: unknown ppn (whatever) detected, using ppn=1",
"Warning: unknown ppn (woohoo) detected, using ppn=1",
],
msg="expected warnings after processing examples %s" % (warnings),
)
reset_warnings()
def test_nodesfilter(self):
"""Test parse_resources_nodes"""
for testdata in RESOURCE_NODES:
orig = testdata.pop('orig')
resources = {}
nodes = parse_resources_nodes(orig, 'gengar', resources)
self.assertEqual(
'nodes=' + resources['nodes'],
nodes,
msg=
'nodes returned is equal to nodes=resources[nodes] (nodes=%s == %s)'
% (resources['nodes'], nodes))
self.assertEqual(
resources,
testdata,
msg=
'generated resources equal to expected (expected %s generated %s; orig %s)'
% (testdata, resources, orig))
# up to now, 2 warnings
warnings = get_warnings()
self.assertEqual(warnings, [
'Warning: unknown ppn (whatever) detected, using ppn=1',
'Warning: unknown ppn (woohoo) detected, using ppn=1',
],
msg="expected warnings after processing examples %s" %
(warnings))
reset_warnings()
def test_make_new_header_pmem_limits(self, mock_cluster_overhead, mock_clusterdata):
sf = SubmitFilter(
[],
[x + "\n" for x in SCRIPTS[6].split("\n")]
)
mock_clusterdata.return_value = {
'TOTMEM': 4096 << 20,
'PHYSMEM': 3072 << 20,
'NP': 8,
'NP_LCD': 2,
}
mock_cluster_overhead.return_value = 0
sf.parse_header()
header = submitfilter.make_new_header(sf)
self.assertEqual(header, [
'#!/bin/bash',
'#PBS -l nodes=1:ppn=4',
'#PBS -l pmem=1g',
'#PBS -m n'
'',
'',
], msg='header with existing mem set')
self.assertEqual(get_warnings(), [
"Unable to determine clustername, using default delcatty (no PBS_DEFAULT)",
"Warning, requested %sb pmem per node, this is more than the available pmem (%sb), this job will never start." % (1 << 30, (3072 << 20) / 8 )
])
def test_cluster_from_options(self):
"""Test cluster from options with ugent/vsc regexp"""
self.assertEqual(cluster_from_options([
('l', 'x'),
('q', '*****@*****.**'),
('m', 'more'),
('q', '*****@*****.**'),
('N', None)
], MASTER_REGEXP), 'whatever',
msg="found expected cluster based on last queue option")
os.environ['PBS_DEFAULT'] = "master10.mycluster.gent.vsc"
self.assertEqual(cluster_from_options([
('l', 'x'),
('q', 'long'),
('m', 'more'),
('q', 'short'),
('N', None)
], MASTER_REGEXP), 'mycluster',
msg="found expected cluster based on PBS_DEFAULT environment variable %s" % os.environ['PBS_DEFAULT'])
self.assertEqual(get_warnings(), [], msg='no cluster warnings')
os.environ['PBS_DEFAULT'] = "master10.notgent.vsc"
self.assertEqual(cluster_from_options([
('l', 'x'),
('q', 'long'),
('m', 'more'),
('q', 'short'),
('N', None)
], MASTER_REGEXP), DEFAULT_SERVER_CLUSTER,
msg="no cluster found, fallback to DEFAULT_SERVER_CLUSTER %s" % DEFAULT_SERVER_CLUSTER)
self.assertEqual(get_warnings(), [
'Unable to determine clustername, using default delcatty (queue short, PBS_DEFAULT %s)' % os.environ['PBS_DEFAULT'],
], msg='no cluster found, warnings generated %s' %(get_warnings()))
del os.environ['PBS_DEFAULT']
def test_cluster_from_options(self):
"""Test cluster from options with ugent/vsc regexp"""
self.assertEqual(
cluster_from_options([('l', 'x'),
('q', '*****@*****.**'),
('m', 'more'),
('q', '*****@*****.**'),
('N', None)], MASTER_REGEXP),
'whatever',
msg="found expected cluster based on last queue option")
os.environ['PBS_DEFAULT'] = "master10.mycluster.gent.vsc"
self.assertEqual(
cluster_from_options([('l', 'x'), ('q', 'long'), ('m', 'more'),
('q', 'short'), ('N', None)], MASTER_REGEXP),
'mycluster',
msg=
"found expected cluster based on PBS_DEFAULT environment variable %s"
% os.environ['PBS_DEFAULT'])
self.assertEqual(get_warnings(), [], msg='no cluster warnings')
os.environ['PBS_DEFAULT'] = "master10.notgent.vsc"
self.assertEqual(
cluster_from_options([('l', 'x'), ('q', 'long'), ('m', 'more'),
('q', 'short'), ('N', None)], MASTER_REGEXP),
DEFAULT_SERVER_CLUSTER,
msg="no cluster found, fallback to DEFAULT_SERVER_CLUSTER %s" %
DEFAULT_SERVER_CLUSTER)
self.assertEqual(get_warnings(), [
'Unable to determine clustername, using default delcatty (queue short, PBS_DEFAULT %s)'
% os.environ['PBS_DEFAULT'],
],
msg='no cluster found, warnings generated %s' %
(get_warnings()))
del os.environ['PBS_DEFAULT']
def test_nodesfilter(self):
"""Test parse_resources_nodes"""
for testdata in RESOURCE_NODES:
orig = testdata.pop('orig')
resources = {}
nodes = parse_resources_nodes(orig, 'gengar', resources)
self.assertEqual('nodes=' + resources['nodes'], nodes,
msg='nodes returned is equal to nodes=resources[nodes] (nodes=%s == %s)' % (resources['nodes'], nodes))
self.assertEqual(resources, testdata, msg='generated resources equal to expected (expected %s generated %s; orig %s)' %
(testdata, resources, orig))
# up to now, 2 warnings
warnings = get_warnings()
self.assertEqual(warnings,
['Warning: unknown ppn (whatever) detected, using ppn=1',
'Warning: unknown ppn (woohoo) detected, using ppn=1',
], msg="expected warnings after processing examples %s" % (warnings))
reset_warnings()
def test_make_new_header_warn(self):
"""
Test make_new_header warnings
ideal ppn
vmem too high
"""
reset_warnings()
sf = SubmitFilter(
[],
[x + "\n" for x in SCRIPTS[3].split("\n")]
)
sf.parse_header()
header = submitfilter.make_new_header(sf)
self.assertEqual(header, sf.header, msg='unmodified header')
self.assertEqual(get_warnings(), [
'The chosen ppn 4 is not considered ideal: should use either lower than or multiple of 3',
'Warning, requested 1099511627776b vmem per node, this is more than the available vmem (86142287872b), this job will never start.',
], msg='warnings for ideal ppn and vmem too high')
def test_make_new_header_warn(self):
"""
Test make_new_header warnings
ideal ppn
vmem too high
"""
reset_warnings()
sf = SubmitFilter(
[],
[x + "\n" for x in SCRIPTS[3].split("\n")]
)
sf.parse_header()
header = submitfilter.make_new_header(sf)
self.assertEqual(header, sf.header, msg='unmodified header')
self.assertEqual(get_warnings(), [
'The chosen ppn 4 is not considered ideal: should use either lower than or multiple of 3',
'Warning, requested 1099511627776b vmem per node, this is more than the available vmem (86142287872b), this job will never start.',
], msg='warnings for ideal ppn and vmem too high')
def main(arguments=None):
"""Main function"""
if arguments is None:
arguments = sys.argv
# This error could otherwise result in empty PBS_O_WORKDIR
try:
os.getcwd()
except OSError as e:
sys.stderr.write("ERROR: Unable to determine current workdir: %s (PWD deleted?)." % e)
sys.stderr.flush()
sys.exit(1)
sf = SubmitFilter(arguments, sys.stdin.readline)
sf.parse_header()
header = make_new_header(sf)
# flush it so it doesn't get mixed with stderr
sys.stdout.flush()
sys.stderr.flush()
# prebody is not stripped of the newline
sys.stdout.write("\n".join(header+[sf.prebody]))
for line in sf.stdin:
sys.stdout.write(line)
# print all generated warnings
# flush it so it doesn't get mixed with stderr
sys.stdout.flush()
for warning in ["%s\n" % w for w in get_warnings()]:
sys.stderr.write(warning)
sys.stderr.flush()
sys.exit(0)
def main(arguments=None):
"""Main function"""
if arguments is None:
arguments = sys.argv
# This error could otherwise result in empty PBS_O_WORKDIR
try:
os.getcwd()
except OSError as e:
sys.stderr.write("ERROR: Unable to determine current workdir: %s (PWD deleted?)." % e)
sys.stderr.flush()
sys.exit(1)
sf = SubmitFilter(arguments, sys.stdin.readline)
sf.parse_header()
header = make_new_header(sf)
# flush it so it doesn't get mixed with stderr
sys.stdout.flush()
sys.stderr.flush()
# prebody is not stripped of the newline
sys.stdout.write("\n".join(header+[sf.prebody]))
for line in sf.stdin:
sys.stdout.write(line)
# print all generated warnings
# flush it so it doesn't get mixed with stderr
sys.stdout.flush()
for warning in ["%s\n" % w for w in get_warnings()]:
sys.stderr.write(warning)
sys.stderr.flush()
sys.exit(0)