Exemplo n.º 1
0
def make_new_header(sf):
    """
    Generate a new header by rewriting selected options and adding missing ones.

    Takes a submitfilter instance as only argument,
    returns the header as a list of strings (one line per element)
    """
    state, newopts = sf.gather_state(MASTER_REGEXP)

    ppn = state['l'].get('_ppn', 1)
    make = sf.make_header

    # make a copy, leave original untouched
    header = sf.header[:]

    # resources: rewrite all resource lines
    for (opt, orig), idx, new in zip(sf.allopts, sf.occur, newopts):
        if opt == 'l' and idx is not None:
            header[idx] = header[idx].replace(orig, new)

    # fix missing
    #
    #    mail: force no mail when no mail is specified
    if 'm' not in state:
        header.extend([
            "# No mail specified - added by submitfilter",
            make("-m", "n"),
        ])

    current_user = pwd.getpwuid(os.getuid()).pw_name

    # vmem: add default when not specified
    if VMEM not in state['l'] and PMEM not in state['l'] and MEM not in state['l']:
        (_, vpp) = get_cluster_mpp(state['_cluster'])
        vmem = vpp * ppn
        state['l'].update({
            VMEM: "%s" % vmem,
            '_%s' % VMEM: vmem,
        })
        header.extend([
            "# No pmem or vmem limit specified - added by submitfilter (server found: %s)" % state['_cluster'],
            make("-l", "%s=%s" % (VMEM, vmem)),
        ])
        syslogger.warn("submitfilter - no [vp]mem specified by user %s. adding %s", current_user, vmem)
    else:
        try:
            requested_memory = (VMEM, state['l'][VMEM])
        except KeyError:
            try:
                requested_memory = (PMEM, state['l'][PMEM])
            except KeyError:
                requested_memory = (MEM, state['l'][MEM])

        syslogger.warn("submitfilter - %s requested by user %s was %s",
                       requested_memory[0], current_user, requested_memory[1])

    #  check whether VSC_NODE_PARTITION environment variable is set
    if 'VSC_NODE_PARTITION' in os.environ:
        header.extend([
            "# Adding PARTITION as specified in VSC_NODE_PARTITION",
            make("-W", "x=PARTITION:%s" % os.environ['VSC_NODE_PARTITION']),
        ])

    # test/warn:
    cl_data = get_clusterdata(state['_cluster'])

    #    cores on cluster: warn when non-ideal number of cores is used (eg 8 cores on 6-core numa domain etc)
    #    ideal: either less than NP_LCD or multiple of NP_LCD
    np_lcd = cl_data['NP_LCD']

    if ppn > np_lcd and ppn % np_lcd:
        warn('The chosen ppn %s is not considered ideal: should use either lower than or multiple of %s' %
             (ppn, np_lcd))

    #    vmem too high: job will not start
    overhead = get_cluster_overhead(state['_cluster'])
    availmem = cl_data['TOTMEM'] - overhead
    if state['l'].get('_%s' % VMEM) > availmem:
        warn("Warning, requested %sb vmem per node, this is more than the available vmem (%sb), this"
             " job will never start." % (state['l']['_%s' % VMEM], availmem))

    #    TODO: mem too low on big-memory systems ?

    return header
Exemplo n.º 2
0
def make_new_header(sf):
    """
    Generate a new header by rewriting selected options and add missing ones.

    Takes a submitfilter instance as only argument,
    returns the header as a list of strings (one line per element)
    """

    # very VSC specific (or only ugent?)
    master_reg = re.compile(r'master[^.]*\.([^.]+)\.(?:[^.]+\.vsc|os)$')

    state, newopts = sf.gather_state(MASTER_REGEXP)

    ppn = state['l'].get('_ppn', 1)
    make = sf.make_header

    # make a copy, leave original untouched
    header = sf.header[:]

    # resources: rewrite all resource lines
    for (opt, orig), idx, new in zip(sf.allopts, sf.occur, newopts):
        if opt == 'l' and idx is not None:
            header[idx] = header[idx].replace(orig, new)

    # fix missing
    #
    #    mail: force no mail when no mail is specified
    if 'm' not in state:
        header.extend([
            "# No mail specified - added by submitfilter",
            make("-m","n"),
        ])

    #    vmem: add default when not specified
    if not 'vmem' in state['l']:
        (ppp, vpp) = get_cluster_mpp(state['_cluster'])
        vmem = vpp * ppn
        state['l'].update({
            'vmem': "%s" % vmem,
            '_vmem': vmem,
        })
        header.extend([
            "# No vmem limit specified - added by submitfilter (server found: %s)" % state['_cluster'],
            make("-l", "vmem=%s" % vmem),
        ])

    #    check whether VSC_NODE_PARTITION environment variable is set
    if 'VSC_NODE_PARTITION' in os.environ:
        header.extend([
            "# Adding PARTITION as specified in VSC_NODE_PARTITION",
            make("-W", "x=PARTITION:%s" % os.environ['VSC_NODE_PARTITION']),
        ])

    # test/warn:
    cl_data = get_clusterdata(state['_cluster'])

    #    cores on cluster: warn when non-ideal number of cores is used (eg 8 cores on 6-core numa domain etc)
    #    ideal: either less than NP_LCD or multiple of NP_LCD
    np_lcd = cl_data['NP_LCD']

    if ppn > np_lcd and ppn % np_lcd:
        warn('The chosen ppn %s is not considered ideal: should use either lower than or multiple of %s' %
             (ppn, np_lcd))

    #    vmem too high: job will not start
    if state['l'].get('_vmem') > cl_data['TOTMEM']:
        warn("Warning, requested %sb vmem per node, this is more than the available vmem (%sb), this"
             " job will never start." % (state['l']['_vmem'], cl_data['TOTMEM']))

    #    TODO: mem too low on big-memory systems ?

    return header
Exemplo n.º 3
0
def make_new_header(sf):
    """
    Generate a new header by rewriting selected options and adding missing ones.

    Takes a submitfilter instance as only argument,
    returns the header as a list of strings (one line per element)
    """
    state, newopts = sf.gather_state(MASTER_REGEXP)

    ppn = state['l'].get('_ppn', 1)
    make = sf.make_header

    # make a copy, leave original untouched
    header = sf.header[:]

    # resources: rewrite all resource lines
    for (opt, orig), idx, new in zip(sf.allopts, sf.occur, newopts):
        if opt == 'l' and idx is not None:
            header[idx] = header[idx].replace(orig, new)

    # fix missing
    #
    #    mail: force no mail when no mail is specified
    if 'm' not in state:
        header.extend([
            "# No mail specified - added by submitfilter",
            make("-m", "n"),
        ])

    current_user = pwd.getpwuid(os.getuid()).pw_name

    # vmem: add default when not specified
    if VMEM not in state['l'] and PMEM not in state['l'] and MEM not in state['l']:
        (_, vpp) = get_cluster_mpp(state['_cluster'])
        vmem = vpp * ppn
        state['l'].update({
            VMEM: "%s" % vmem,
            '_%s' % VMEM: vmem,
        })
        header.extend([
            "# No pmem or vmem limit specified - added by submitfilter (server found: %s)" % state['_cluster'],
            make("-l", "%s=%s" % (VMEM, vmem)),
        ])
        logging.warn("submitfilter - no [vp]mem specified by user %s. adding %s", current_user, vmem)
    else:
        try:
            requested_memory = (VMEM, state['l'][VMEM])
        except KeyError:
            try:
                requested_memory = (PMEM, state['l'][PMEM])
            except KeyError:
                requested_memory = (MEM, state['l'][MEM])

        logging.info("submitfilter - %s requested by user %s was %s",
                       requested_memory[0], current_user, requested_memory[1])

    #  check whether VSC_NODE_PARTITION environment variable is set
    if ENV_NODE_PARTITION in os.environ:
        header.extend([
            "# Adding PARTITION as specified in %s" % ENV_NODE_PARTITION,
            make("-W", "x=PARTITION:%s" % os.environ[ENV_NODE_PARTITION]),
        ])

    #  check whether VSC_RESERVATION environment variable is set
    if ENV_RESERVATION in os.environ:
        header.extend([
            "# Adding reservation as specified in %s" % ENV_RESERVATION,
            make("-W", "x=FLAGS:ADVRES:%s" % os.environ[ENV_RESERVATION]),
        ])

    # test/warn:
    cl_data = get_clusterdata(state['_cluster'])

    #    cores on cluster: warn when non-ideal number of cores is used (eg 8 cores on 6-core numa domain etc)
    #    ideal: either less than NP_LCD or multiple of NP_LCD
    np_lcd = cl_data['NP_LCD']

    if ppn > np_lcd and ppn % np_lcd:
        warn('The chosen ppn %s is not considered ideal: should use either lower than or multiple of %s' %
             (ppn, np_lcd))

    # vmem, mem, pmem too high: job will not start
    overhead = get_cluster_overhead(state['_cluster'])
    availmem = cl_data['TOTMEM'] - overhead
    physmem = cl_data['PHYSMEM'] - overhead
    if state['l'].get('_%s' % VMEM) > availmem:
        requested = state['l'].get('_%s' % VMEM) or state['l'].get('_%s' % MEM)
        warn("Warning, requested %sb vmem per node, this is more than the available vmem (%sb), this"
             " job will never start." % (requested, availmem))
    elif state['l'].get('_%s' % MEM) > physmem:
        requested = state['l'].get('_%s' % MEM)
        warn("Warning, requested %sb mem per node, this is more than the available mem (%sb), this"
             " job will never start." % (requested, physmem))
    elif state['l'].get('_%s' % PMEM) > physmem / cl_data['NP']:
        requested = state['l'].get('_%s' % PMEM)
        warn("Warning, requested %sb pmem per node, this is more than the available pmem (%sb), this"
             " job will never start." % (requested, physmem / cl_data['NP']))

    return header