def test_maxppn(self):
"""Test get_cluster_maxppn"""
for cluster in CLUSTERDATA.keys():
cd=CLUSTERDATA[cluster]
self.assertEqual(get_cluster_maxppn(cluster),
cd['NP'] if 'NP' in cd else cd['DEFMAXNP'],
msg="Found expected maxppn for cluster %s" % cluster)
def parse_mem(name, txt, cluster, resources):
"""
Convert <name:(p|v|pv)?mem>=<txt> for cluster
update resources instance with
_<name>: value in bytes
<name>: (possibly modified) resource text
returns (possibly modified) resource text
Supported modifications:
(v|p)mem=all/full ; (v|p)mem=half
"""
req_in_bytes = _parse_mem_units(txt)
if req_in_bytes is None:
(ppp, vpp) = get_cluster_mpp(cluster)
maxppn = get_cluster_maxppn(cluster)
convert = {
PMEM: maxppn * ppp,
VMEM: maxppn * vpp,
}
# multiplier 1 == identity op
multi = lambda x: x
if name not in (PMEM, VMEM, MEM):
# TODO: and do what? use pmem?
warn('Unsupported memory specification %s with value %s' %
(name, txt))
elif txt == 'half':
multi = lambda x: int(x / 2)
# default to pmem
req_in_bytes = multi(convert.get(name, convert[PMEM]))
txt = "%s" % req_in_bytes
resources.update({
name: txt, # original notation if possible
"_%s" % name: req_in_bytes,
})
return "%s=%s" % (name, txt)
def parse_mem(name, txt, cluster, resources):
"""
Convert <name:(p|v|pv)?mem>=<txt> for cluster
update resources instance with
_<name>: value in bytes
<name>: (possibly modified) resource text
returns (possibly modified) resource text
Supported modifications:
(v|p)mem=all/full ; (v|p)mem=half
"""
req_in_bytes = _parse_mem_units(txt)
if req_in_bytes is None:
(ppp, vpp) = get_cluster_mpp(cluster)
maxppn = get_cluster_maxppn(cluster)
convert = {
PMEM: maxppn * ppp,
VMEM: maxppn * vpp,
}
# multiplier 1 == identity op
multi = lambda x: x
if name not in (PMEM, VMEM, MEM):
# TODO: and do what? use pmem?
warn('Unsupported memory specification %s with value %s' % (name, txt))
elif txt == 'half':
multi = lambda x: int(x/2)
# default to pmem
req_in_bytes = multi(convert.get(name, convert[PMEM]))
txt = "%s" % req_in_bytes
resources.update({
name: txt, # original notation if possible
"_%s" % name: req_in_bytes,
})
return "%s=%s" % (name, txt)
def parse_resources_nodes(txt, cluster, resources):
"""
Convert -l nodes=<txt> for cluster
update resources instance with
_nrnodes: number of nodes
_nrcores: total cores
_ppn: (avg) ppn
nodes: (possibly modified) resource text
returns (possibly modified) resource text
Supported modifications:
ppn=all/full ; ppn=half
"""
# syntax is a +-separated node_spec
# each node_spec has id[:prop[:prop[:...]]]
# id is integer (=number of nodes) or a single node name
# property: either special ppn=integer or something else/arbitrary
maxppn = get_cluster_maxppn(cluster)
nrnodes = 0
nrcores = 0
newtxt = []
for node_spec in txt.split('+'):
props = node_spec.split(':')
ppns = [(x.split('=')[1], idx) for idx, x in enumerate(props) if x.startswith('ppn=')] or [(1, None)]
ppn = ppns[0][0]
try:
ppn = int(ppn)
except ValueError:
if ppn in ('all', 'full',):
ppn = maxppn
elif ppn == 'half':
ppn = max(1, int(maxppn / 2))
else:
# it's ok to always warn for this
# (even if it is not the final used option)
warn("Warning: unknown ppn (%s) detected, using ppn=1" % (ppn,))
ppn = 1
ppntxt = 'ppn=%s' % ppn
if ppns[0][1] is None:
props.append(ppntxt)
else:
props[ppns[0][1]] = ppntxt
try:
nodes = int(props[0])
except (ValueError, IndexError):
# some description
nodes = 1
nrnodes += nodes
nrcores += nodes * ppn
newtxt.append(':'.join(props))
# update shared resources dict
resources.update({
'_nrnodes': nrnodes,
'_nrcores': nrcores,
'_ppn': max(1, int(nrcores / nrnodes)),
NODES_PREFIX: '+'.join(newtxt),
})
return "%s=%s" % (NODES_PREFIX, resources[NODES_PREFIX])
def parse_resources_nodes(txt, cluster, resources):
"""
Convert -l nodes=<txt> for cluster
update resources instance with
_nrnodes: number of nodes
_nrcores: total cores
_ppn: (avg) ppn
nodes: (possibly modified) resource text
returns (possibly modified) resource text
Supported modifications:
ppn=all/full ; ppn=half
"""
# syntax is a +-separated node_spec
# each node_spec has id[:prop[:prop[:...]]]
# id is integer (=number of nodes) or a single node name
# property: either special ppn=integer or something else/arbitrary
maxppn = get_cluster_maxppn(cluster)
nrnodes = 0
nrcores = 0
newtxt = []
for node_spec in txt.split('+'):
props = node_spec.split(':')
ppns = [(x.split('=')[1], idx) for idx, x in enumerate(props)
if x.startswith('ppn=')] or [(1, None)]
ppn = ppns[0][0]
try:
ppn = int(ppn)
except ValueError:
if ppn in (
'all',
'full',
):
ppn = maxppn
elif ppn == 'half':
ppn = max(1, int(maxppn / 2))
else:
# it's ok to always warn for this
# (even if it is not the final used option)
warn("Warning: unknown ppn (%s) detected, using ppn=1" %
(ppn, ))
ppn = 1
ppntxt = 'ppn=%s' % ppn
if ppns[0][1] is None:
props.append(ppntxt)
else:
props[ppns[0][1]] = ppntxt
try:
nodes = int(props[0])
except (ValueError, IndexError):
# some description
nodes = 1
nrnodes += nodes
nrcores += nodes * ppn
newtxt.append(':'.join(props))
# update shared resources dict
resources.update({
'_nrnodes': nrnodes,
'_nrcores': nrcores,
'_ppn': max(1, int(nrcores / nrnodes)),
NODES_PREFIX: '+'.join(newtxt),
})
return "%s=%s" % (NODES_PREFIX, resources[NODES_PREFIX])
