def test_maxppn(self): """Test get_cluster_maxppn""" for cluster in CLUSTERDATA.keys(): cd=CLUSTERDATA[cluster] self.assertEqual(get_cluster_maxppn(cluster), cd['NP'] if 'NP' in cd else cd['DEFMAXNP'], msg="Found expected maxppn for cluster %s" % cluster)
def parse_mem(name, txt, cluster, resources): """ Convert <name:(p|v|pv)?mem>=<txt> for cluster update resources instance with _<name>: value in bytes <name>: (possibly modified) resource text returns (possibly modified) resource text Supported modifications: (v|p)mem=all/full ; (v|p)mem=half """ req_in_bytes = _parse_mem_units(txt) if req_in_bytes is None: (ppp, vpp) = get_cluster_mpp(cluster) maxppn = get_cluster_maxppn(cluster) convert = { PMEM: maxppn * ppp, VMEM: maxppn * vpp, } # multiplier 1 == identity op multi = lambda x: x if name not in (PMEM, VMEM, MEM): # TODO: and do what? use pmem? warn('Unsupported memory specification %s with value %s' % (name, txt)) elif txt == 'half': multi = lambda x: int(x / 2) # default to pmem req_in_bytes = multi(convert.get(name, convert[PMEM])) txt = "%s" % req_in_bytes resources.update({ name: txt, # original notation if possible "_%s" % name: req_in_bytes, }) return "%s=%s" % (name, txt)
def parse_mem(name, txt, cluster, resources): """ Convert <name:(p|v|pv)?mem>=<txt> for cluster update resources instance with _<name>: value in bytes <name>: (possibly modified) resource text returns (possibly modified) resource text Supported modifications: (v|p)mem=all/full ; (v|p)mem=half """ req_in_bytes = _parse_mem_units(txt) if req_in_bytes is None: (ppp, vpp) = get_cluster_mpp(cluster) maxppn = get_cluster_maxppn(cluster) convert = { PMEM: maxppn * ppp, VMEM: maxppn * vpp, } # multiplier 1 == identity op multi = lambda x: x if name not in (PMEM, VMEM, MEM): # TODO: and do what? use pmem? warn('Unsupported memory specification %s with value %s' % (name, txt)) elif txt == 'half': multi = lambda x: int(x/2) # default to pmem req_in_bytes = multi(convert.get(name, convert[PMEM])) txt = "%s" % req_in_bytes resources.update({ name: txt, # original notation if possible "_%s" % name: req_in_bytes, }) return "%s=%s" % (name, txt)
def parse_resources_nodes(txt, cluster, resources): """ Convert -l nodes=<txt> for cluster update resources instance with _nrnodes: number of nodes _nrcores: total cores _ppn: (avg) ppn nodes: (possibly modified) resource text returns (possibly modified) resource text Supported modifications: ppn=all/full ; ppn=half """ # syntax is a +-separated node_spec # each node_spec has id[:prop[:prop[:...]]] # id is integer (=number of nodes) or a single node name # property: either special ppn=integer or something else/arbitrary maxppn = get_cluster_maxppn(cluster) nrnodes = 0 nrcores = 0 newtxt = [] for node_spec in txt.split('+'): props = node_spec.split(':') ppns = [(x.split('=')[1], idx) for idx, x in enumerate(props) if x.startswith('ppn=')] or [(1, None)] ppn = ppns[0][0] try: ppn = int(ppn) except ValueError: if ppn in ('all', 'full',): ppn = maxppn elif ppn == 'half': ppn = max(1, int(maxppn / 2)) else: # it's ok to always warn for this # (even if it is not the final used option) warn("Warning: unknown ppn (%s) detected, using ppn=1" % (ppn,)) ppn = 1 ppntxt = 'ppn=%s' % ppn if ppns[0][1] is None: props.append(ppntxt) else: props[ppns[0][1]] = ppntxt try: nodes = int(props[0]) except (ValueError, IndexError): # some description nodes = 1 nrnodes += nodes nrcores += nodes * ppn newtxt.append(':'.join(props)) # update shared resources dict resources.update({ '_nrnodes': nrnodes, '_nrcores': nrcores, '_ppn': max(1, int(nrcores / nrnodes)), NODES_PREFIX: '+'.join(newtxt), }) return "%s=%s" % (NODES_PREFIX, resources[NODES_PREFIX])
def parse_resources_nodes(txt, cluster, resources): """ Convert -l nodes=<txt> for cluster update resources instance with _nrnodes: number of nodes _nrcores: total cores _ppn: (avg) ppn nodes: (possibly modified) resource text returns (possibly modified) resource text Supported modifications: ppn=all/full ; ppn=half """ # syntax is a +-separated node_spec # each node_spec has id[:prop[:prop[:...]]] # id is integer (=number of nodes) or a single node name # property: either special ppn=integer or something else/arbitrary maxppn = get_cluster_maxppn(cluster) nrnodes = 0 nrcores = 0 newtxt = [] for node_spec in txt.split('+'): props = node_spec.split(':') ppns = [(x.split('=')[1], idx) for idx, x in enumerate(props) if x.startswith('ppn=')] or [(1, None)] ppn = ppns[0][0] try: ppn = int(ppn) except ValueError: if ppn in ( 'all', 'full', ): ppn = maxppn elif ppn == 'half': ppn = max(1, int(maxppn / 2)) else: # it's ok to always warn for this # (even if it is not the final used option) warn("Warning: unknown ppn (%s) detected, using ppn=1" % (ppn, )) ppn = 1 ppntxt = 'ppn=%s' % ppn if ppns[0][1] is None: props.append(ppntxt) else: props[ppns[0][1]] = ppntxt try: nodes = int(props[0]) except (ValueError, IndexError): # some description nodes = 1 nrnodes += nodes nrcores += nodes * ppn newtxt.append(':'.join(props)) # update shared resources dict resources.update({ '_nrnodes': nrnodes, '_nrcores': nrcores, '_ppn': max(1, int(nrcores / nrnodes)), NODES_PREFIX: '+'.join(newtxt), }) return "%s=%s" % (NODES_PREFIX, resources[NODES_PREFIX])