def get_nodes_dict():
"""Get the pbs_nodes equivalent info as dict"""
query = get_query()
node_states = query.getnodes([])
for name, full_state in node_states.items():
# just add states
states = full_state[ATTR_STATE]
if ND_free in states and ATTR_JOBS in full_state:
_log.debug('Added free_and_job node %s' % (name))
states.insert(0, ND_free_and_job)
if ND_free in states and not ATTR_JOBS in full_state:
_log.debug('Append idle node %s' % (name))
states.append(ND_idle) # append it, not insert
if ATTR_ERROR in full_state:
_log.debug('Added error node %s' % (name))
states.insert(0, ND_error)
if ND_down in states and ATTR_ERROR in full_state:
_log.debug('Added down_on_error node %s' % (name))
states.insert(0, ND_down_on_error)
# extend the node dict with derived dict (for convenience)
derived = {}
if ATTR_JOBS in full_state:
jobs = full_state.get_jobs()
if not all([JOBID_REG.search(x.strip()) for x in jobs]):
_log.debug('Added bad node %s for jobs %s' % (name, jobs))
states.insert(0, ND_bad)
derived[ATTR_JOBS] = jobs
derived[ATTR_STATES] = [str(x) for x in states]
make_state_map(derived)
if ATTR_NP in full_state:
derived[ATTR_NP] = int(full_state[ATTR_NP][0])
if ATTR_STATUS in full_state:
status = full_state[ATTR_STATUS]
for prop in ['physmem', 'totmem', 'size']:
if not prop in status:
continue
val = status.get(prop)[0]
if prop in ('size',):
# 'size': ['539214180kb:539416640kb']
# - use 2nd field
val = val.split(':')[1]
derived[prop] = str2byte(val)
full_state['derived'] = derived
_log.debug("node %s derived data %s " % (name, derived))
return node_states
def det_common_path_prefix(paths):
"""Determine common path prefix for a given list of paths."""
if not isinstance(paths, list):
_log.error("det_common_path_prefix: argument must be of type list (got %s: %s)" % (type(paths), paths))
elif not paths:
return None
# initial guess for common prefix
prefix = paths[0]
found_common = False
while not found_common and prefix != os.path.dirname(prefix):
prefix = os.path.dirname(prefix)
found_common = all([p.startswith(prefix) for p in paths])
if found_common:
# prefix may be empty string for relative paths with a non-common prefix
return prefix.rstrip(os.path.sep) or None
else:
return None
def det_common_path_prefix(paths):
"""Determine common path prefix for a given list of paths."""
if not isinstance(paths, list):
_log.error(
"det_common_path_prefix: argument must be of type list (got %s: %s)"
% (type(paths), paths))
elif not paths:
return None
# initial guess for common prefix
prefix = paths[0]
found_common = False
while not found_common and prefix != os.path.dirname(prefix):
prefix = os.path.dirname(prefix)
found_common = all([p.startswith(prefix) for p in paths])
if found_common:
# prefix may be empty string for relative paths with a non-common prefix
return prefix.rstrip(os.path.sep) or None
else:
return None
def collect_nodeinfo():
"""Collect node information"""
types = {}
state_list = []
node_list = []
re_host_id = re.compile(r"(?P<id>\d+)")
for idx, (node, full_state) in enumerate(get_nodes()):
# A node can have serveral states. We are only interested in first entry.
derived = full_state['derived']
# what state to report?
state_list.append(derived['state'])
if derived['nodestate'] == NDST_OK:
cores = derived.get('np', None)
physmem = derived.get('physmem', None)
totmem = derived.get('totmem', None)
size = derived.get('size', None)
if all([cores, physmem, totmem, size]): # there shouldn't be any value 0
# round mem to 1 gb, size to 5gb
GB = str2byte('gb')
pmem = ceil(10 * physmem / GB) / 10
tmem = ceil(10 * totmem / GB) / 10
swap = tmem - pmem
dsize = ceil(10 * size / (5 * GB)) / 2
typ = (cores, pmem, swap, dsize)
if not typ in types:
types[typ] = []
types[typ].append(node)
result = re_host_id.search(node)
if result:
node_list.append(result.group('id'))
else:
node_list.append(str(idx + 1)) # offset +1
return node_list, state_list, types
def prepare(self, onlymod=None):
"""
Prepare a set of environment parameters based on name/version of toolchain
- load modules for toolchain and dependencies
- generate extra variables and set them in the environment
onlymod: Boolean/string to indicate if the toolchain should only load the environment
with module (True) or also set all other variables (False) like compiler CC etc
(If string: comma separated list of variables that will be ignored).
"""
if self.modules_tool is None:
self.log.raiseException("No modules tool defined.")
if not self._toolchain_exists():
self.log.raiseException("No module found for toolchain name '%s' (%s)" % (self.name, self.version))
if self.name == DUMMY_TOOLCHAIN_NAME:
if self.version == DUMMY_TOOLCHAIN_VERSION:
self.log.info('prepare: toolchain dummy mode, dummy version; not loading dependencies')
else:
self.log.info('prepare: toolchain dummy mode and loading dependencies')
self.modules_tool.load([dep['short_mod_name'] for dep in self.dependencies])
return
# Load the toolchain and dependencies modules
self.log.debug("Loading toolchain module and dependencies...")
# make sure toolchain is available using short module name by running 'module use' on module path subdir
if self.init_modpaths:
mod_path_suffix = build_option('suffix_modules_path')
for modpath in self.init_modpaths:
self.modules_tool.prepend_module_path(os.path.join(install_path('mod'), mod_path_suffix, modpath))
self.modules_tool.load([self.det_short_module_name()])
self.modules_tool.load([dep['short_mod_name'] for dep in self.dependencies])
# determine direct toolchain dependencies
mod_name = self.det_short_module_name()
self.toolchain_dep_mods = self.modules_tool.dependencies_for(mod_name, depth=0)
self.log.debug('prepare: list of direct toolchain dependencies: %s' % self.toolchain_dep_mods)
# only retain names of toolchain elements, excluding toolchain name
toolchain_definition = set([e for es in self.definition().values() for e in es if not e == self.name])
# filter out optional toolchain elements if they're not used in the module
for elem_name in toolchain_definition.copy():
if self.is_required(elem_name) or self.is_dep_in_toolchain_module(elem_name):
continue
# not required and missing: remove from toolchain definition
self.log.debug("Removing %s from list of optional toolchain elements." % elem_name)
toolchain_definition.remove(elem_name)
self.log.debug("List of toolchain dependencies from toolchain module: %s" % self.toolchain_dep_mods)
self.log.debug("List of toolchain elements from toolchain definition: %s" % toolchain_definition)
if all(map(self.is_dep_in_toolchain_module, toolchain_definition)):
self.log.info("List of toolchain dependency modules and toolchain definition match!")
else:
self.log.error("List of toolchain dependency modules and toolchain definition do not match " \
"(%s vs %s)" % (self.toolchain_dep_mods, toolchain_definition))
# Generate the variables to be set
self.set_variables()
# set the variables
# onlymod can be comma-separated string of variables not to be set
if onlymod == True:
self.log.debug("prepare: do not set additional variables onlymod=%s" % onlymod)
self.generate_vars()
else:
self.log.debug("prepare: set additional variables onlymod=%s" % onlymod)
# add LDFLAGS and CPPFLAGS from dependencies to self.vars
self._add_dependency_variables()
self.generate_vars()
self._setenv_variables(onlymod)
def prepare(self, onlymod=None):
"""
Prepare a set of environment parameters based on name/version of toolchain
- load modules for toolchain and dependencies
- generate extra variables and set them in the environment
onlymod: Boolean/string to indicate if the toolchain should only load the environment
with module (True) or also set all other variables (False) like compiler CC etc
(If string: comma separated list of variables that will be ignored).
"""
if self.modules_tool is None:
self.log.raiseException("No modules tool defined.")
if not self._toolchain_exists():
self.log.raiseException(
"No module found for toolchain name '%s' (%s)" %
(self.name, self.version))
if self.name == DUMMY_TOOLCHAIN_NAME:
if self.version == DUMMY_TOOLCHAIN_VERSION:
self.log.info(
'prepare: toolchain dummy mode, dummy version; not loading dependencies'
)
else:
self.log.info(
'prepare: toolchain dummy mode and loading dependencies')
self.modules_tool.load(
[dep['short_mod_name'] for dep in self.dependencies])
return
# Load the toolchain and dependencies modules
self.log.debug("Loading toolchain module and dependencies...")
# make sure toolchain is available using short module name by running 'module use' on module path subdir
if self.init_modpaths:
mod_path_suffix = build_option('suffix_modules_path')
for modpath in self.init_modpaths:
self.modules_tool.prepend_module_path(
os.path.join(install_path('mod'), mod_path_suffix,
modpath))
self.modules_tool.load([self.det_short_module_name()])
self.modules_tool.load(
[dep['short_mod_name'] for dep in self.dependencies])
# determine direct toolchain dependencies
mod_name = self.det_short_module_name()
self.toolchain_dep_mods = self.modules_tool.dependencies_for(mod_name,
depth=0)
self.log.debug('prepare: list of direct toolchain dependencies: %s' %
self.toolchain_dep_mods)
# only retain names of toolchain elements, excluding toolchain name
toolchain_definition = set([
e for es in self.definition().values() for e in es
if not e == self.name
])
# filter out optional toolchain elements if they're not used in the module
for elem_name in toolchain_definition.copy():
if self.is_required(elem_name) or self.is_dep_in_toolchain_module(
elem_name):
continue
# not required and missing: remove from toolchain definition
self.log.debug(
"Removing %s from list of optional toolchain elements." %
elem_name)
toolchain_definition.remove(elem_name)
self.log.debug(
"List of toolchain dependencies from toolchain module: %s" %
self.toolchain_dep_mods)
self.log.debug(
"List of toolchain elements from toolchain definition: %s" %
toolchain_definition)
if all(map(self.is_dep_in_toolchain_module, toolchain_definition)):
self.log.info(
"List of toolchain dependency modules and toolchain definition match!"
)
else:
self.log.error("List of toolchain dependency modules and toolchain definition do not match " \
"(%s vs %s)" % (self.toolchain_dep_mods, toolchain_definition))
# Generate the variables to be set
self.set_variables()
# set the variables
# onlymod can be comma-separated string of variables not to be set
if onlymod == True:
self.log.debug(
"prepare: do not set additional variables onlymod=%s" %
onlymod)
self.generate_vars()
else:
self.log.debug("prepare: set additional variables onlymod=%s" %
onlymod)
# add LDFLAGS and CPPFLAGS from dependencies to self.vars
self._add_dependency_variables()
self.generate_vars()
self._setenv_variables(onlymod)
