def get_nodes_dict(): """Get the pbs_nodes equivalent info as dict""" query = get_query() node_states = query.getnodes([]) for name, full_state in node_states.items(): # just add states states = full_state[ATTR_STATE] if ND_free in states and ATTR_JOBS in full_state: _log.debug('Added free_and_job node %s' % (name)) states.insert(0, ND_free_and_job) if ND_free in states and not ATTR_JOBS in full_state: _log.debug('Append idle node %s' % (name)) states.append(ND_idle) # append it, not insert if ATTR_ERROR in full_state: _log.debug('Added error node %s' % (name)) states.insert(0, ND_error) if ND_down in states and ATTR_ERROR in full_state: _log.debug('Added down_on_error node %s' % (name)) states.insert(0, ND_down_on_error) # extend the node dict with derived dict (for convenience) derived = {} if ATTR_JOBS in full_state: jobs = full_state.get_jobs() if not all([JOBID_REG.search(x.strip()) for x in jobs]): _log.debug('Added bad node %s for jobs %s' % (name, jobs)) states.insert(0, ND_bad) derived[ATTR_JOBS] = jobs derived[ATTR_STATES] = [str(x) for x in states] make_state_map(derived) if ATTR_NP in full_state: derived[ATTR_NP] = int(full_state[ATTR_NP][0]) if ATTR_STATUS in full_state: status = full_state[ATTR_STATUS] for prop in ['physmem', 'totmem', 'size']: if not prop in status: continue val = status.get(prop)[0] if prop in ('size',): # 'size': ['539214180kb:539416640kb'] # - use 2nd field val = val.split(':')[1] derived[prop] = str2byte(val) full_state['derived'] = derived _log.debug("node %s derived data %s " % (name, derived)) return node_states
def det_common_path_prefix(paths): """Determine common path prefix for a given list of paths.""" if not isinstance(paths, list): _log.error("det_common_path_prefix: argument must be of type list (got %s: %s)" % (type(paths), paths)) elif not paths: return None # initial guess for common prefix prefix = paths[0] found_common = False while not found_common and prefix != os.path.dirname(prefix): prefix = os.path.dirname(prefix) found_common = all([p.startswith(prefix) for p in paths]) if found_common: # prefix may be empty string for relative paths with a non-common prefix return prefix.rstrip(os.path.sep) or None else: return None
def det_common_path_prefix(paths): """Determine common path prefix for a given list of paths.""" if not isinstance(paths, list): _log.error( "det_common_path_prefix: argument must be of type list (got %s: %s)" % (type(paths), paths)) elif not paths: return None # initial guess for common prefix prefix = paths[0] found_common = False while not found_common and prefix != os.path.dirname(prefix): prefix = os.path.dirname(prefix) found_common = all([p.startswith(prefix) for p in paths]) if found_common: # prefix may be empty string for relative paths with a non-common prefix return prefix.rstrip(os.path.sep) or None else: return None
def collect_nodeinfo(): """Collect node information""" types = {} state_list = [] node_list = [] re_host_id = re.compile(r"(?P<id>\d+)") for idx, (node, full_state) in enumerate(get_nodes()): # A node can have serveral states. We are only interested in first entry. derived = full_state['derived'] # what state to report? state_list.append(derived['state']) if derived['nodestate'] == NDST_OK: cores = derived.get('np', None) physmem = derived.get('physmem', None) totmem = derived.get('totmem', None) size = derived.get('size', None) if all([cores, physmem, totmem, size]): # there shouldn't be any value 0 # round mem to 1 gb, size to 5gb GB = str2byte('gb') pmem = ceil(10 * physmem / GB) / 10 tmem = ceil(10 * totmem / GB) / 10 swap = tmem - pmem dsize = ceil(10 * size / (5 * GB)) / 2 typ = (cores, pmem, swap, dsize) if not typ in types: types[typ] = [] types[typ].append(node) result = re_host_id.search(node) if result: node_list.append(result.group('id')) else: node_list.append(str(idx + 1)) # offset +1 return node_list, state_list, types
def prepare(self, onlymod=None): """ Prepare a set of environment parameters based on name/version of toolchain - load modules for toolchain and dependencies - generate extra variables and set them in the environment onlymod: Boolean/string to indicate if the toolchain should only load the environment with module (True) or also set all other variables (False) like compiler CC etc (If string: comma separated list of variables that will be ignored). """ if self.modules_tool is None: self.log.raiseException("No modules tool defined.") if not self._toolchain_exists(): self.log.raiseException("No module found for toolchain name '%s' (%s)" % (self.name, self.version)) if self.name == DUMMY_TOOLCHAIN_NAME: if self.version == DUMMY_TOOLCHAIN_VERSION: self.log.info('prepare: toolchain dummy mode, dummy version; not loading dependencies') else: self.log.info('prepare: toolchain dummy mode and loading dependencies') self.modules_tool.load([dep['short_mod_name'] for dep in self.dependencies]) return # Load the toolchain and dependencies modules self.log.debug("Loading toolchain module and dependencies...") # make sure toolchain is available using short module name by running 'module use' on module path subdir if self.init_modpaths: mod_path_suffix = build_option('suffix_modules_path') for modpath in self.init_modpaths: self.modules_tool.prepend_module_path(os.path.join(install_path('mod'), mod_path_suffix, modpath)) self.modules_tool.load([self.det_short_module_name()]) self.modules_tool.load([dep['short_mod_name'] for dep in self.dependencies]) # determine direct toolchain dependencies mod_name = self.det_short_module_name() self.toolchain_dep_mods = self.modules_tool.dependencies_for(mod_name, depth=0) self.log.debug('prepare: list of direct toolchain dependencies: %s' % self.toolchain_dep_mods) # only retain names of toolchain elements, excluding toolchain name toolchain_definition = set([e for es in self.definition().values() for e in es if not e == self.name]) # filter out optional toolchain elements if they're not used in the module for elem_name in toolchain_definition.copy(): if self.is_required(elem_name) or self.is_dep_in_toolchain_module(elem_name): continue # not required and missing: remove from toolchain definition self.log.debug("Removing %s from list of optional toolchain elements." % elem_name) toolchain_definition.remove(elem_name) self.log.debug("List of toolchain dependencies from toolchain module: %s" % self.toolchain_dep_mods) self.log.debug("List of toolchain elements from toolchain definition: %s" % toolchain_definition) if all(map(self.is_dep_in_toolchain_module, toolchain_definition)): self.log.info("List of toolchain dependency modules and toolchain definition match!") else: self.log.error("List of toolchain dependency modules and toolchain definition do not match " \ "(%s vs %s)" % (self.toolchain_dep_mods, toolchain_definition)) # Generate the variables to be set self.set_variables() # set the variables # onlymod can be comma-separated string of variables not to be set if onlymod == True: self.log.debug("prepare: do not set additional variables onlymod=%s" % onlymod) self.generate_vars() else: self.log.debug("prepare: set additional variables onlymod=%s" % onlymod) # add LDFLAGS and CPPFLAGS from dependencies to self.vars self._add_dependency_variables() self.generate_vars() self._setenv_variables(onlymod)
def prepare(self, onlymod=None): """ Prepare a set of environment parameters based on name/version of toolchain - load modules for toolchain and dependencies - generate extra variables and set them in the environment onlymod: Boolean/string to indicate if the toolchain should only load the environment with module (True) or also set all other variables (False) like compiler CC etc (If string: comma separated list of variables that will be ignored). """ if self.modules_tool is None: self.log.raiseException("No modules tool defined.") if not self._toolchain_exists(): self.log.raiseException( "No module found for toolchain name '%s' (%s)" % (self.name, self.version)) if self.name == DUMMY_TOOLCHAIN_NAME: if self.version == DUMMY_TOOLCHAIN_VERSION: self.log.info( 'prepare: toolchain dummy mode, dummy version; not loading dependencies' ) else: self.log.info( 'prepare: toolchain dummy mode and loading dependencies') self.modules_tool.load( [dep['short_mod_name'] for dep in self.dependencies]) return # Load the toolchain and dependencies modules self.log.debug("Loading toolchain module and dependencies...") # make sure toolchain is available using short module name by running 'module use' on module path subdir if self.init_modpaths: mod_path_suffix = build_option('suffix_modules_path') for modpath in self.init_modpaths: self.modules_tool.prepend_module_path( os.path.join(install_path('mod'), mod_path_suffix, modpath)) self.modules_tool.load([self.det_short_module_name()]) self.modules_tool.load( [dep['short_mod_name'] for dep in self.dependencies]) # determine direct toolchain dependencies mod_name = self.det_short_module_name() self.toolchain_dep_mods = self.modules_tool.dependencies_for(mod_name, depth=0) self.log.debug('prepare: list of direct toolchain dependencies: %s' % self.toolchain_dep_mods) # only retain names of toolchain elements, excluding toolchain name toolchain_definition = set([ e for es in self.definition().values() for e in es if not e == self.name ]) # filter out optional toolchain elements if they're not used in the module for elem_name in toolchain_definition.copy(): if self.is_required(elem_name) or self.is_dep_in_toolchain_module( elem_name): continue # not required and missing: remove from toolchain definition self.log.debug( "Removing %s from list of optional toolchain elements." % elem_name) toolchain_definition.remove(elem_name) self.log.debug( "List of toolchain dependencies from toolchain module: %s" % self.toolchain_dep_mods) self.log.debug( "List of toolchain elements from toolchain definition: %s" % toolchain_definition) if all(map(self.is_dep_in_toolchain_module, toolchain_definition)): self.log.info( "List of toolchain dependency modules and toolchain definition match!" ) else: self.log.error("List of toolchain dependency modules and toolchain definition do not match " \ "(%s vs %s)" % (self.toolchain_dep_mods, toolchain_definition)) # Generate the variables to be set self.set_variables() # set the variables # onlymod can be comma-separated string of variables not to be set if onlymod == True: self.log.debug( "prepare: do not set additional variables onlymod=%s" % onlymod) self.generate_vars() else: self.log.debug("prepare: set additional variables onlymod=%s" % onlymod) # add LDFLAGS and CPPFLAGS from dependencies to self.vars self._add_dependency_variables() self.generate_vars() self._setenv_variables(onlymod)