예제 #1
0
def parse_mem(name, txt, cluster, resources):
    """
    Convert <name:(p|v|pv)?mem>=<txt> for cluster

    update resources instance with
        _<name>: value in bytes
        <name>: (possibly modified) resource text

    returns (possibly modified) resource text

    Supported modifications:
        (v|p)mem=all/full ; (v|p)mem=half
    """
    req_in_bytes = _parse_mem_units(txt)

    if req_in_bytes is None:
        (ppp, vpp) = get_cluster_mpp(cluster)
        maxppn = get_cluster_maxppn(cluster)

        convert = {
            PMEM: maxppn * ppp,
            VMEM: maxppn * vpp,
        }

        # multiplier 1 == identity op
        multi = lambda x: x
        if name not in (PMEM, VMEM, MEM):
            # TODO: and do what? use pmem?
            warn('Unsupported memory specification %s with value %s' %
                 (name, txt))
        elif txt == 'half':
            multi = lambda x: int(x / 2)

        # default to pmem
        req_in_bytes = multi(convert.get(name, convert[PMEM]))
        txt = "%s" % req_in_bytes

    resources.update({
        name: txt,  # original notation if possible
        "_%s" % name: req_in_bytes,
    })

    return "%s=%s" % (name, txt)
예제 #2
0
def parse_mem(name, txt, cluster, resources):
    """
    Convert <name:(p|v|pv)?mem>=<txt> for cluster

    update resources instance with
        _<name>: value in bytes
        <name>: (possibly modified) resource text

    returns (possibly modified) resource text

    Supported modifications:
        (v|p)mem=all/full ; (v|p)mem=half
    """
    req_in_bytes = _parse_mem_units(txt)

    if req_in_bytes is None:
        (ppp, vpp) = get_cluster_mpp(cluster)
        maxppn = get_cluster_maxppn(cluster)

        convert = {
            PMEM: maxppn * ppp,
            VMEM: maxppn * vpp,
        }

        # multiplier 1 == identity op
        multi = lambda x: x
        if name not in (PMEM, VMEM, MEM):
            # TODO: and do what? use pmem?
            warn('Unsupported memory specification %s with value %s' % (name, txt))
        elif txt == 'half':
            multi = lambda x: int(x/2)

        # default to pmem
        req_in_bytes = multi(convert.get(name, convert[PMEM]))
        txt = "%s" % req_in_bytes

    resources.update({
        name: txt,  # original notation if possible
        "_%s" % name: req_in_bytes,
    })

    return "%s=%s" % (name, txt)
예제 #3
0
 def test_mpp(self):
     """Test get_cluster_vpp"""
     for cl, mpp in [('delcatty', (4049213952, 4720302336)),
                     ('zzzmytest', (2116052906, 2339749077)),
                     ('zzzmytestavail', (1902719573, 2126415744))]:
         self.assertEqual(get_cluster_mpp(cl), mpp, msg="expected mpp %s for %s" % (mpp, cl,))
예제 #4
0
def make_new_header(sf):
    """
    Generate a new header by rewriting selected options and add missing ones.

    Takes a submitfilter instance as only argument,
    returns the header as a list of strings (one line per element)
    """

    # very VSC specific (or only ugent?)
    master_reg = re.compile(r'master[^.]*\.([^.]+)\.(?:[^.]+\.vsc|os)$')

    state, newopts = sf.gather_state(MASTER_REGEXP)

    ppn = state['l'].get('_ppn', 1)
    make = sf.make_header

    # make a copy, leave original untouched
    header = sf.header[:]

    # resources: rewrite all resource lines
    for (opt, orig), idx, new in zip(sf.allopts, sf.occur, newopts):
        if opt == 'l' and idx is not None:
            header[idx] = header[idx].replace(orig, new)

    # fix missing
    #
    #    mail: force no mail when no mail is specified
    if 'm' not in state:
        header.extend([
            "# No mail specified - added by submitfilter",
            make("-m","n"),
        ])

    #    vmem: add default when not specified
    if not 'vmem' in state['l']:
        (ppp, vpp) = get_cluster_mpp(state['_cluster'])
        vmem = vpp * ppn
        state['l'].update({
            'vmem': "%s" % vmem,
            '_vmem': vmem,
        })
        header.extend([
            "# No vmem limit specified - added by submitfilter (server found: %s)" % state['_cluster'],
            make("-l", "vmem=%s" % vmem),
        ])

    #    check whether VSC_NODE_PARTITION environment variable is set
    if 'VSC_NODE_PARTITION' in os.environ:
        header.extend([
            "# Adding PARTITION as specified in VSC_NODE_PARTITION",
            make("-W", "x=PARTITION:%s" % os.environ['VSC_NODE_PARTITION']),
        ])

    # test/warn:
    cl_data = get_clusterdata(state['_cluster'])

    #    cores on cluster: warn when non-ideal number of cores is used (eg 8 cores on 6-core numa domain etc)
    #    ideal: either less than NP_LCD or multiple of NP_LCD
    np_lcd = cl_data['NP_LCD']

    if ppn > np_lcd and ppn % np_lcd:
        warn('The chosen ppn %s is not considered ideal: should use either lower than or multiple of %s' %
             (ppn, np_lcd))

    #    vmem too high: job will not start
    if state['l'].get('_vmem') > cl_data['TOTMEM']:
        warn("Warning, requested %sb vmem per node, this is more than the available vmem (%sb), this"
             " job will never start." % (state['l']['_vmem'], cl_data['TOTMEM']))

    #    TODO: mem too low on big-memory systems ?

    return header
예제 #5
0
 def test_mpp(self):
     """Test get_cluster_vpp"""
     for cl, mpp in [('delcatty', (4226900480, 4897988864)), ('zzzmytest', (2116052906, 2339749077))]:
         self.assertEqual(get_cluster_mpp(cl), mpp, msg="expected mpp %s for %s" % (mpp, cl,))
예제 #6
0
def make_new_header(sf):
    """
    Generate a new header by rewriting selected options and adding missing ones.

    Takes a submitfilter instance as only argument,
    returns the header as a list of strings (one line per element)
    """
    state, newopts = sf.gather_state(MASTER_REGEXP)

    ppn = state['l'].get('_ppn', 1)
    make = sf.make_header

    # make a copy, leave original untouched
    header = sf.header[:]

    # resources: rewrite all resource lines
    for (opt, orig), idx, new in zip(sf.allopts, sf.occur, newopts):
        if opt == 'l' and idx is not None:
            header[idx] = header[idx].replace(orig, new)

    # fix missing
    #
    #    mail: force no mail when no mail is specified
    if 'm' not in state:
        header.extend([
            "# No mail specified - added by submitfilter",
            make("-m", "n"),
        ])

    current_user = pwd.getpwuid(os.getuid()).pw_name

    # vmem: add default when not specified
    if VMEM not in state['l'] and PMEM not in state['l'] and MEM not in state['l']:
        (_, vpp) = get_cluster_mpp(state['_cluster'])
        vmem = vpp * ppn
        state['l'].update({
            VMEM: "%s" % vmem,
            '_%s' % VMEM: vmem,
        })
        header.extend([
            "# No pmem or vmem limit specified - added by submitfilter (server found: %s)" % state['_cluster'],
            make("-l", "%s=%s" % (VMEM, vmem)),
        ])
        syslogger.warn("submitfilter - no [vp]mem specified by user %s. adding %s", current_user, vmem)
    else:
        try:
            requested_memory = (VMEM, state['l'][VMEM])
        except KeyError:
            try:
                requested_memory = (PMEM, state['l'][PMEM])
            except KeyError:
                requested_memory = (MEM, state['l'][MEM])

        syslogger.warn("submitfilter - %s requested by user %s was %s",
                       requested_memory[0], current_user, requested_memory[1])

    #  check whether VSC_NODE_PARTITION environment variable is set
    if 'VSC_NODE_PARTITION' in os.environ:
        header.extend([
            "# Adding PARTITION as specified in VSC_NODE_PARTITION",
            make("-W", "x=PARTITION:%s" % os.environ['VSC_NODE_PARTITION']),
        ])

    # test/warn:
    cl_data = get_clusterdata(state['_cluster'])

    #    cores on cluster: warn when non-ideal number of cores is used (eg 8 cores on 6-core numa domain etc)
    #    ideal: either less than NP_LCD or multiple of NP_LCD
    np_lcd = cl_data['NP_LCD']

    if ppn > np_lcd and ppn % np_lcd:
        warn('The chosen ppn %s is not considered ideal: should use either lower than or multiple of %s' %
             (ppn, np_lcd))

    #    vmem too high: job will not start
    overhead = get_cluster_overhead(state['_cluster'])
    availmem = cl_data['TOTMEM'] - overhead
    if state['l'].get('_%s' % VMEM) > availmem:
        warn("Warning, requested %sb vmem per node, this is more than the available vmem (%sb), this"
             " job will never start." % (state['l']['_%s' % VMEM], availmem))

    #    TODO: mem too low on big-memory systems ?

    return header
예제 #7
0
def make_new_header(sf):
    """
    Generate a new header by rewriting selected options and adding missing ones.

    Takes a submitfilter instance as only argument,
    returns the header as a list of strings (one line per element)
    """
    state, newopts = sf.gather_state(MASTER_REGEXP)

    ppn = state['l'].get('_ppn', 1)
    make = sf.make_header

    # make a copy, leave original untouched
    header = sf.header[:]

    # resources: rewrite all resource lines
    for (opt, orig), idx, new in zip(sf.allopts, sf.occur, newopts):
        if opt == 'l' and idx is not None:
            header[idx] = header[idx].replace(orig, new)

    # fix missing
    #
    #    mail: force no mail when no mail is specified
    if 'm' not in state:
        header.extend([
            "# No mail specified - added by submitfilter",
            make("-m", "n"),
        ])

    current_user = pwd.getpwuid(os.getuid()).pw_name

    # vmem: add default when not specified
    if VMEM not in state['l'] and PMEM not in state['l'] and MEM not in state['l']:
        (_, vpp) = get_cluster_mpp(state['_cluster'])
        vmem = vpp * ppn
        state['l'].update({
            VMEM: "%s" % vmem,
            '_%s' % VMEM: vmem,
        })
        header.extend([
            "# No pmem or vmem limit specified - added by submitfilter (server found: %s)" % state['_cluster'],
            make("-l", "%s=%s" % (VMEM, vmem)),
        ])
        logging.warn("submitfilter - no [vp]mem specified by user %s. adding %s", current_user, vmem)
    else:
        try:
            requested_memory = (VMEM, state['l'][VMEM])
        except KeyError:
            try:
                requested_memory = (PMEM, state['l'][PMEM])
            except KeyError:
                requested_memory = (MEM, state['l'][MEM])

        logging.info("submitfilter - %s requested by user %s was %s",
                       requested_memory[0], current_user, requested_memory[1])

    #  check whether VSC_NODE_PARTITION environment variable is set
    if ENV_NODE_PARTITION in os.environ:
        header.extend([
            "# Adding PARTITION as specified in %s" % ENV_NODE_PARTITION,
            make("-W", "x=PARTITION:%s" % os.environ[ENV_NODE_PARTITION]),
        ])

    #  check whether VSC_RESERVATION environment variable is set
    if ENV_RESERVATION in os.environ:
        header.extend([
            "# Adding reservation as specified in %s" % ENV_RESERVATION,
            make("-W", "x=FLAGS:ADVRES:%s" % os.environ[ENV_RESERVATION]),
        ])

    # test/warn:
    cl_data = get_clusterdata(state['_cluster'])

    #    cores on cluster: warn when non-ideal number of cores is used (eg 8 cores on 6-core numa domain etc)
    #    ideal: either less than NP_LCD or multiple of NP_LCD
    np_lcd = cl_data['NP_LCD']

    if ppn > np_lcd and ppn % np_lcd:
        warn('The chosen ppn %s is not considered ideal: should use either lower than or multiple of %s' %
             (ppn, np_lcd))

    # vmem, mem, pmem too high: job will not start
    overhead = get_cluster_overhead(state['_cluster'])
    availmem = cl_data['TOTMEM'] - overhead
    physmem = cl_data['PHYSMEM'] - overhead
    if state['l'].get('_%s' % VMEM) > availmem:
        requested = state['l'].get('_%s' % VMEM) or state['l'].get('_%s' % MEM)
        warn("Warning, requested %sb vmem per node, this is more than the available vmem (%sb), this"
             " job will never start." % (requested, availmem))
    elif state['l'].get('_%s' % MEM) > physmem:
        requested = state['l'].get('_%s' % MEM)
        warn("Warning, requested %sb mem per node, this is more than the available mem (%sb), this"
             " job will never start." % (requested, physmem))
    elif state['l'].get('_%s' % PMEM) > physmem / cl_data['NP']:
        requested = state['l'].get('_%s' % PMEM)
        warn("Warning, requested %sb pmem per node, this is more than the available pmem (%sb), this"
             " job will never start." % (requested, physmem / cl_data['NP']))

    return header