def test_getdata(self):
"""Test parse_resources_nodes"""
self.assertEqual(get_clusterdata('doesnotexists'), CLUSTERDATA[DEFAULT_SERVER_CLUSTER],
msg='get_clusterdata for non-existing cluster returns DEFAULT_SERVER_CLUSTER data')
for cluster in SORTED_CLUSTERS:
self.assertEqual(get_clusterdata(cluster), CLUSTERDATA[cluster],
msg='get_clusterdata returns correct data for clsuetr %s' %cluster)
def make_new_header(sf):
"""
Generate a new header by rewriting selected options and adding missing ones.
Takes a submitfilter instance as only argument,
returns the header as a list of strings (one line per element)
"""
state, newopts = sf.gather_state(MASTER_REGEXP)
ppn = state['l'].get('_ppn', 1)
make = sf.make_header
# make a copy, leave original untouched
header = sf.header[:]
# resources: rewrite all resource lines
for (opt, orig), idx, new in zip(sf.allopts, sf.occur, newopts):
if opt == 'l' and idx is not None:
header[idx] = header[idx].replace(orig, new)
# fix missing
#
# mail: force no mail when no mail is specified
if 'm' not in state:
header.extend([
"# No mail specified - added by submitfilter",
make("-m", "n"),
])
current_user = pwd.getpwuid(os.getuid()).pw_name
# vmem: add default when not specified
if VMEM not in state['l'] and PMEM not in state['l'] and MEM not in state['l']:
(_, vpp) = get_cluster_mpp(state['_cluster'])
vmem = vpp * ppn
state['l'].update({
VMEM: "%s" % vmem,
'_%s' % VMEM: vmem,
})
header.extend([
"# No pmem or vmem limit specified - added by submitfilter (server found: %s)" % state['_cluster'],
make("-l", "%s=%s" % (VMEM, vmem)),
])
syslogger.warn("submitfilter - no [vp]mem specified by user %s. adding %s", current_user, vmem)
else:
try:
requested_memory = (VMEM, state['l'][VMEM])
except KeyError:
try:
requested_memory = (PMEM, state['l'][PMEM])
except KeyError:
requested_memory = (MEM, state['l'][MEM])
syslogger.warn("submitfilter - %s requested by user %s was %s",
requested_memory[0], current_user, requested_memory[1])
# check whether VSC_NODE_PARTITION environment variable is set
if 'VSC_NODE_PARTITION' in os.environ:
header.extend([
"# Adding PARTITION as specified in VSC_NODE_PARTITION",
make("-W", "x=PARTITION:%s" % os.environ['VSC_NODE_PARTITION']),
])
# test/warn:
cl_data = get_clusterdata(state['_cluster'])
# cores on cluster: warn when non-ideal number of cores is used (eg 8 cores on 6-core numa domain etc)
# ideal: either less than NP_LCD or multiple of NP_LCD
np_lcd = cl_data['NP_LCD']
if ppn > np_lcd and ppn % np_lcd:
warn('The chosen ppn %s is not considered ideal: should use either lower than or multiple of %s' %
(ppn, np_lcd))
# vmem too high: job will not start
overhead = get_cluster_overhead(state['_cluster'])
availmem = cl_data['TOTMEM'] - overhead
if state['l'].get('_%s' % VMEM) > availmem:
warn("Warning, requested %sb vmem per node, this is more than the available vmem (%sb), this"
" job will never start." % (state['l']['_%s' % VMEM], availmem))
# TODO: mem too low on big-memory systems ?
return header
def make_new_header(sf):
"""
Generate a new header by rewriting selected options and add missing ones.
Takes a submitfilter instance as only argument,
returns the header as a list of strings (one line per element)
"""
# very VSC specific (or only ugent?)
master_reg = re.compile(r'master[^.]*\.([^.]+)\.(?:[^.]+\.vsc|os)$')
state, newopts = sf.gather_state(MASTER_REGEXP)
ppn = state['l'].get('_ppn', 1)
make = sf.make_header
# make a copy, leave original untouched
header = sf.header[:]
# resources: rewrite all resource lines
for (opt, orig), idx, new in zip(sf.allopts, sf.occur, newopts):
if opt == 'l' and idx is not None:
header[idx] = header[idx].replace(orig, new)
# fix missing
#
# mail: force no mail when no mail is specified
if 'm' not in state:
header.extend([
"# No mail specified - added by submitfilter",
make("-m","n"),
])
# vmem: add default when not specified
if not 'vmem' in state['l']:
(ppp, vpp) = get_cluster_mpp(state['_cluster'])
vmem = vpp * ppn
state['l'].update({
'vmem': "%s" % vmem,
'_vmem': vmem,
})
header.extend([
"# No vmem limit specified - added by submitfilter (server found: %s)" % state['_cluster'],
make("-l", "vmem=%s" % vmem),
])
# check whether VSC_NODE_PARTITION environment variable is set
if 'VSC_NODE_PARTITION' in os.environ:
header.extend([
"# Adding PARTITION as specified in VSC_NODE_PARTITION",
make("-W", "x=PARTITION:%s" % os.environ['VSC_NODE_PARTITION']),
])
# test/warn:
cl_data = get_clusterdata(state['_cluster'])
# cores on cluster: warn when non-ideal number of cores is used (eg 8 cores on 6-core numa domain etc)
# ideal: either less than NP_LCD or multiple of NP_LCD
np_lcd = cl_data['NP_LCD']
if ppn > np_lcd and ppn % np_lcd:
warn('The chosen ppn %s is not considered ideal: should use either lower than or multiple of %s' %
(ppn, np_lcd))
# vmem too high: job will not start
if state['l'].get('_vmem') > cl_data['TOTMEM']:
warn("Warning, requested %sb vmem per node, this is more than the available vmem (%sb), this"
" job will never start." % (state['l']['_vmem'], cl_data['TOTMEM']))
# TODO: mem too low on big-memory systems ?
return header
def make_new_header(sf):
"""
Generate a new header by rewriting selected options and adding missing ones.
Takes a submitfilter instance as only argument,
returns the header as a list of strings (one line per element)
"""
state, newopts = sf.gather_state(MASTER_REGEXP)
ppn = state['l'].get('_ppn', 1)
make = sf.make_header
# make a copy, leave original untouched
header = sf.header[:]
# resources: rewrite all resource lines
for (opt, orig), idx, new in zip(sf.allopts, sf.occur, newopts):
if opt == 'l' and idx is not None:
header[idx] = header[idx].replace(orig, new)
# fix missing
#
# mail: force no mail when no mail is specified
if 'm' not in state:
header.extend([
"# No mail specified - added by submitfilter",
make("-m", "n"),
])
current_user = pwd.getpwuid(os.getuid()).pw_name
# vmem: add default when not specified
if VMEM not in state['l'] and PMEM not in state['l'] and MEM not in state['l']:
(_, vpp) = get_cluster_mpp(state['_cluster'])
vmem = vpp * ppn
state['l'].update({
VMEM: "%s" % vmem,
'_%s' % VMEM: vmem,
})
header.extend([
"# No pmem or vmem limit specified - added by submitfilter (server found: %s)" % state['_cluster'],
make("-l", "%s=%s" % (VMEM, vmem)),
])
logging.warn("submitfilter - no [vp]mem specified by user %s. adding %s", current_user, vmem)
else:
try:
requested_memory = (VMEM, state['l'][VMEM])
except KeyError:
try:
requested_memory = (PMEM, state['l'][PMEM])
except KeyError:
requested_memory = (MEM, state['l'][MEM])
logging.info("submitfilter - %s requested by user %s was %s",
requested_memory[0], current_user, requested_memory[1])
# check whether VSC_NODE_PARTITION environment variable is set
if ENV_NODE_PARTITION in os.environ:
header.extend([
"# Adding PARTITION as specified in %s" % ENV_NODE_PARTITION,
make("-W", "x=PARTITION:%s" % os.environ[ENV_NODE_PARTITION]),
])
# check whether VSC_RESERVATION environment variable is set
if ENV_RESERVATION in os.environ:
header.extend([
"# Adding reservation as specified in %s" % ENV_RESERVATION,
make("-W", "x=FLAGS:ADVRES:%s" % os.environ[ENV_RESERVATION]),
])
# test/warn:
cl_data = get_clusterdata(state['_cluster'])
# cores on cluster: warn when non-ideal number of cores is used (eg 8 cores on 6-core numa domain etc)
# ideal: either less than NP_LCD or multiple of NP_LCD
np_lcd = cl_data['NP_LCD']
if ppn > np_lcd and ppn % np_lcd:
warn('The chosen ppn %s is not considered ideal: should use either lower than or multiple of %s' %
(ppn, np_lcd))
# vmem, mem, pmem too high: job will not start
overhead = get_cluster_overhead(state['_cluster'])
availmem = cl_data['TOTMEM'] - overhead
physmem = cl_data['PHYSMEM'] - overhead
if state['l'].get('_%s' % VMEM) > availmem:
requested = state['l'].get('_%s' % VMEM) or state['l'].get('_%s' % MEM)
warn("Warning, requested %sb vmem per node, this is more than the available vmem (%sb), this"
" job will never start." % (requested, availmem))
elif state['l'].get('_%s' % MEM) > physmem:
requested = state['l'].get('_%s' % MEM)
warn("Warning, requested %sb mem per node, this is more than the available mem (%sb), this"
" job will never start." % (requested, physmem))
elif state['l'].get('_%s' % PMEM) > physmem / cl_data['NP']:
requested = state['l'].get('_%s' % PMEM)
warn("Warning, requested %sb pmem per node, this is more than the available pmem (%sb), this"
" job will never start." % (requested, physmem / cl_data['NP']))
return header