def det_toolchain_element_details(tc, elem): """ Determine details of a particular toolchain element, for a given Toolchain instance. """ # check for cached version first tc_dict = tc.as_dict() key = (tc_dict['name'], tc_dict['version'] + tc_dict['versionsuffix'], elem) if key in _toolchain_details_cache: _log.debug("Obtained details for '%s' in toolchain '%s' from cache" % (elem, tc_dict)) return _toolchain_details_cache[key] # grab version from parsed easyconfig file for toolchain eb_file = robot_find_easyconfig(tc_dict['name'], det_full_ec_version(tc_dict)) tc_ec = process_easyconfig(eb_file, parse_only=True) if len(tc_ec) > 1: _log.warning("More than one toolchain specification found for %s, only retaining first" % tc_dict) _log.debug("Full list of toolchain specifications: %s" % tc_ec) tc_ec = tc_ec[0]['ec'] tc_deps = tc_ec['dependencies'] tc_elem_details = None for tc_dep in tc_deps: if tc_dep['name'] == elem: tc_elem_details = tc_dep _log.debug("Found details for toolchain element %s: %s" % (elem, tc_elem_details)) break if tc_elem_details is None: # for compiler-only toolchains, toolchain and compilers are one-and-the-same if tc_ec['name'] == elem: tc_elem_details = tc_ec else: raise EasyBuildError("No toolchain element '%s' found for toolchain %s: %s", elem, tc.as_dict(), tc_ec) _toolchain_details_cache[key] = tc_elem_details _log.debug("Obtained details for '%s' in toolchain '%s', added to cache" % (elem, tc_dict)) return _toolchain_details_cache[key]
def det_toolchain_element_details(tc, elem): """ Determine details of a particular toolchain element, for a given Toolchain instance. """ # check for cached version first tc_dict = tc.as_dict() key = (tc_dict['name'], tc_dict['version'] + tc_dict['versionsuffix'], elem) if key in _toolchain_details_cache: _log.debug("Obtained details for '%s' in toolchain '%s' from cache" % (elem, tc_dict)) return _toolchain_details_cache[key] # grab version from parsed easyconfig file for toolchain eb_file = robot_find_easyconfig(tc_dict['name'], det_full_ec_version(tc_dict)) tc_ec = process_easyconfig(eb_file, parse_only=True) if len(tc_ec) > 1: _log.warning("More than one toolchain specification found for %s, only retaining first" % tc_dict) _log.debug("Full list of toolchain specifications: %s" % tc_ec) tc_ec = tc_ec[0]['ec'] tc_deps = tc_ec['dependencies'] tc_elem_details = None for tc_dep in tc_deps: if tc_dep['name'] == elem: tc_elem_details = tc_dep _log.debug("Found details for toolchain element %s: %s" % (elem, tc_elem_details)) break if tc_elem_details is None: # for compiler-only toolchains, toolchain and compilers are one-and-the-same if tc_ec['name'] == elem: tc_elem_details = tc_ec else: _log.error("No toolchain element '%s' found for toolchain %s: %s" % (elem, tc.as_dict(), tc_ec)) _toolchain_details_cache[key] = tc_elem_details _log.debug("Obtained details for '%s' in toolchain '%s', added to cache" % (elem, tc_dict)) return _toolchain_details_cache[key]
def robot_find_minimal_easyconfig_for_dependency(dep): """ Find an easyconfig with minimal toolchain for a dependency """ newdep = copy.deepcopy(dep) toolchain_hierarchy = get_toolchain_hierarchy(dep['toolchain']) res = None # reversed search: start with subtoolchains first, i.e. first (dummy or) compiler-only toolchain, etc. for toolchain in toolchain_hierarchy: newdep['toolchain'] = toolchain eb_file = robot_find_easyconfig(newdep['name'], det_full_ec_version(newdep)) if eb_file is not None: if newdep['toolchain'] != dep['toolchain']: _log.info( "Minimally resolving dependency %s using toolchain %s with %s", dep, toolchain, eb_file) res = (newdep, eb_file) break if res is None: _log.debug("Irresolvable minimal dependency found: %s", dep) return res
def robot_find_minimal_easyconfig_for_dependency(dep): """ Find an easyconfig with minimal toolchain for a dependency """ newdep = copy.deepcopy(dep) toolchain_hierarchy = get_toolchain_hierarchy(dep['toolchain']) res = None # reversed search: start with subtoolchains first, i.e. first (dummy or) compiler-only toolchain, etc. for toolchain in toolchain_hierarchy: newdep['toolchain'] = toolchain eb_file = robot_find_easyconfig(newdep['name'], det_full_ec_version(newdep)) if eb_file is not None: if newdep['toolchain'] != dep['toolchain']: _log.info("Minimally resolving dependency %s using toolchain %s with %s", dep, toolchain, eb_file) res = (newdep, eb_file) break if res is None: _log.debug("Irresolvable minimal dependency found: %s", dep) return res
def resolve_dependencies(unprocessed, build_specs=None, retain_all_deps=False): """ Work through the list of easyconfigs to determine an optimal order @param unprocessed: list of easyconfigs @param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...) @param retain_all_deps: boolean indicating whether all dependencies must be retained, regardless of availability; retain all deps when True, check matching build option when False """ robot = build_option('robot_path') # retain all dependencies if specified by either the resp. build option or the dedicated named argument retain_all_deps = build_option('retain_all_deps') or retain_all_deps if retain_all_deps: # assume that no modules are available when forced, to retain all dependencies avail_modules = [] _log.info("Forcing all dependencies to be retained.") else: # Get a list of all available modules (format: [(name, installversion), ...]) avail_modules = modules_tool().available() if len(avail_modules) == 0: _log.warning( "No installed modules. Your MODULEPATH is probably incomplete: %s" % os.getenv('MODULEPATH')) ordered_ecs = [] # all available modules can be used for resolving dependencies except those that will be installed being_installed = [p['full_mod_name'] for p in unprocessed] avail_modules = [m for m in avail_modules if not m in being_installed] _log.debug('unprocessed before resolving deps: %s' % unprocessed) # resolve all dependencies, put a safeguard in place to avoid an infinite loop (shouldn't occur though) irresolvable = [] loopcnt = 0 maxloopcnt = 10000 while unprocessed: # make sure this stops, we really don't want to get stuck in an infinite loop loopcnt += 1 if loopcnt > maxloopcnt: tup = (maxloopcnt, unprocessed, irresolvable) msg = "Maximum loop cnt %s reached, so quitting (unprocessed: %s, irresolvable: %s)" % tup _log.error(msg) # first try resolving dependencies without using external dependencies last_processed_count = -1 while len(avail_modules) > last_processed_count: last_processed_count = len(avail_modules) res = find_resolved_modules(unprocessed, avail_modules, retain_all_deps=retain_all_deps) more_ecs, unprocessed, avail_modules = res for ec in more_ecs: if not ec['full_mod_name'] in [ x['full_mod_name'] for x in ordered_ecs ]: ordered_ecs.append(ec) # robot: look for existing dependencies, add them if robot and unprocessed: # rely on EasyBuild module naming scheme when resolving dependencies, since we know that will # generate sensible module names that include the necessary information for the resolution to work # (name, version, toolchain, versionsuffix) being_installed = [ EasyBuildMNS().det_full_module_name(p['ec']) for p in unprocessed ] additional = [] for entry in unprocessed: # do not choose an entry that is being installed in the current run # if they depend, you probably want to rebuild them using the new dependency deps = entry['dependencies'] candidates = [ d for d in deps if not EasyBuildMNS().det_full_module_name( d) in being_installed ] if candidates: cand_dep = candidates[0] # find easyconfig, might not find any _log.debug("Looking for easyconfig for %s" % str(cand_dep)) # note: robot_find_easyconfig may return None path = robot_find_easyconfig(cand_dep['name'], det_full_ec_version(cand_dep)) if path is None: # no easyconfig found for dependency, add to list of irresolvable dependencies if cand_dep not in irresolvable: _log.debug("Irresolvable dependency found: %s" % cand_dep) irresolvable.append(cand_dep) # remove irresolvable dependency from list of dependencies so we can continue entry['dependencies'].remove(cand_dep) else: _log.info("Robot: resolving dependency %s with %s" % (cand_dep, path)) # build specs should not be passed down to resolved dependencies, # to avoid that e.g. --try-toolchain trickles down into the used toolchain itself hidden = cand_dep.get('hidden', False) processed_ecs = process_easyconfig( path, validate=not retain_all_deps, hidden=hidden) # ensure that selected easyconfig provides required dependency mods = [ spec['ec'].full_mod_name for spec in processed_ecs ] dep_mod_name = ActiveMNS().det_full_module_name( cand_dep) if not dep_mod_name in mods: tup = (path, dep_mod_name, mods) _log.error( "easyconfig file %s does not contain module %s (mods: %s)" % tup) for ec in processed_ecs: if not ec in unprocessed + additional: additional.append(ec) _log.debug("Added %s as dependency of %s" % (ec, entry)) else: mod_name = EasyBuildMNS().det_full_module_name(entry['ec']) _log.debug( "No more candidate dependencies to resolve for %s" % mod_name) # add additional (new) easyconfigs to list of stuff to process unprocessed.extend(additional) elif not robot: # no use in continuing if robot is not enabled, dependencies won't be resolved anyway irresolvable = [ dep for x in unprocessed for dep in x['dependencies'] ] break if irresolvable: _log.warning("Irresolvable dependencies (details): %s" % irresolvable) irresolvable_mods_eb = [ EasyBuildMNS().det_full_module_name(dep) for dep in irresolvable ] _log.warning("Irresolvable dependencies (EasyBuild module names): %s" % ', '.join(irresolvable_mods_eb)) irresolvable_mods = [ ActiveMNS().det_full_module_name(dep) for dep in irresolvable ] _log.error('Irresolvable dependencies encountered: %s' % ', '.join(irresolvable_mods)) _log.info("Dependency resolution complete, building as follows:\n%s" % ordered_ecs) return ordered_ecs
def get_toolchain_hierarchy(parent_toolchain): """ Determine list of subtoolchains for specified parent toolchain. Result starts with the most minimal subtoolchains first, ends with specified toolchain. The dummy toolchain is considered the most minimal subtoolchain only if the add_dummy_to_minimal_toolchains build option is enabled. @param parent_toolchain: dictionary with name/version of parent toolchain """ # obtain list of all possible subtoolchains _, all_tc_classes = search_toolchain('') subtoolchains = dict((tc_class.NAME, getattr(tc_class, 'SUBTOOLCHAIN', None)) for tc_class in all_tc_classes) current_tc_name, current_tc_version = parent_toolchain['name'], parent_toolchain['version'] subtoolchain_name, subtoolchain_version = subtoolchains[current_tc_name], None # the parent toolchain is at the top of the hierarchy toolchain_hierarchy = [parent_toolchain] while subtoolchain_name: # grab the easyconfig of the current toolchain and search the dependencies for a version of the subtoolchain path = robot_find_easyconfig(current_tc_name, current_tc_version) if path is None: raise EasyBuildError("Could not find easyconfig for %(name)s toolchain version %(version)s", current_tc_name, current_tc_version) # parse the easyconfig parsed_ec = process_easyconfig(path)[0] # search the dependencies for the version of the subtoolchain dep_tcs = [dep_toolchain['toolchain'] for dep_toolchain in parsed_ec['dependencies'] if dep_toolchain['toolchain']['name'] == subtoolchain_name] unique_dep_tc_versions = set([dep_tc['version'] for dep_tc in dep_tcs]) if len(unique_dep_tc_versions) == 1: subtoolchain_version = dep_tcs[0]['version'] elif len(unique_dep_tc_versions) == 0: # only retain GCCcore as subtoolchain if version was found if subtoolchain_name == GCCcore.NAME: _log.info("No version found for %s; assuming legacy toolchain and skipping it as subtoolchain.", subtoolchain_name) subtoolchain_name = GCCcore.SUBTOOLCHAIN subtoolchain_version = '' # dummy toolchain: end of the line elif subtoolchain_name == DUMMY_TOOLCHAIN_NAME: subtoolchain_version = '' else: raise EasyBuildError("No version found for subtoolchain %s in dependencies of %s", subtoolchain_name, current_tc_name) else: raise EasyBuildError("Multiple versions of %s found in dependencies of toolchain %s: %s", subtoolchain_name, current_tc_name, unique_dep_tc_versions) if subtoolchain_name == DUMMY_TOOLCHAIN_NAME and not build_option('add_dummy_to_minimal_toolchains'): # we're done break # add to hierarchy and move to next current_tc_name, current_tc_version = subtoolchain_name, subtoolchain_version subtoolchain_name, subtoolchain_version = subtoolchains[current_tc_name], None toolchain_hierarchy.insert(0, {'name': current_tc_name, 'version': current_tc_version}) _log.info("Found toolchain hierarchy for toolchain %s: %s", parent_toolchain, toolchain_hierarchy) return toolchain_hierarchy
def resolve_dependencies(easyconfigs, modtool, retain_all_deps=False, raise_error_missing_ecs=True): """ Work through the list of easyconfigs to determine an optimal order :param easyconfigs: list of easyconfigs :param modtool: ModulesTool instance to use :param retain_all_deps: boolean indicating whether all dependencies must be retained, regardless of availability; retain all deps when True, check matching build option when False :param raise_error_missing_ecs: raise an error when one or more easyconfig files could not be found """ robot = build_option('robot_path') # retain all dependencies if specified by either the resp. build option or the dedicated named argument retain_all_deps = build_option('retain_all_deps') or retain_all_deps avail_modules = modtool.available() if retain_all_deps: # assume that no modules are available when forced, to retain all dependencies avail_modules = [] _log.info("Forcing all dependencies to be retained.") else: if len(avail_modules) == 0: _log.warning( "No installed modules. Your MODULEPATH is probably incomplete: %s" % os.getenv('MODULEPATH')) ordered_ecs = [] # all available modules can be used for resolving dependencies except those that will be installed being_installed = [p['full_mod_name'] for p in easyconfigs] avail_modules = [m for m in avail_modules if m not in being_installed] _log.debug('easyconfigs before resolving deps: %s', easyconfigs) totally_missing, missing_easyconfigs = [], [] # resolve all dependencies, put a safeguard in place to avoid an infinite loop (shouldn't occur though) loopcnt = 0 maxloopcnt = 10000 while easyconfigs: # make sure this stops, we really don't want to get stuck in an infinite loop loopcnt += 1 if loopcnt > maxloopcnt: raise EasyBuildError( "Maximum loop cnt %s reached, so quitting (easyconfigs: %s, missing_easyconfigs: %s)", maxloopcnt, easyconfigs, missing_easyconfigs) # first try resolving dependencies without using external dependencies last_processed_count = -1 while len(avail_modules) > last_processed_count: last_processed_count = len(avail_modules) res = find_resolved_modules(easyconfigs, avail_modules, modtool, retain_all_deps=retain_all_deps) resolved_ecs, easyconfigs, avail_modules = res ordered_ec_mod_names = [x['full_mod_name'] for x in ordered_ecs] for ec in resolved_ecs: # only add easyconfig if it's not included yet (based on module name) if not ec['full_mod_name'] in ordered_ec_mod_names: ordered_ecs.append(ec) # dependencies marked as external modules should be resolved via available modules at this point missing_external_modules = [ d['full_mod_name'] for ec in easyconfigs for d in ec['dependencies'] if d.get('external_module', False) ] if missing_external_modules: raise EasyBuildError( "Missing modules for dependencies marked as external modules: %s", ', '.join(missing_external_modules)) # robot: look for existing dependencies, add them if robot and easyconfigs: # rely on EasyBuild module naming scheme when resolving dependencies, since we know that will # generate sensible module names that include the necessary information for the resolution to work # (name, version, toolchain, versionsuffix) being_installed = [ EasyBuildMNS().det_full_module_name(p['ec']) for p in easyconfigs ] additional = [] for entry in easyconfigs: # do not choose an entry that is being installed in the current run # if they depend, you probably want to rebuild them using the new dependency deps = entry['dependencies'] candidates = [ d for d in deps if not EasyBuildMNS().det_full_module_name( d) in being_installed ] if candidates: cand_dep = candidates[0] # find easyconfig, might not find any _log.debug("Looking for easyconfig for %s" % str(cand_dep)) # note: robot_find_easyconfig may return None path = robot_find_easyconfig(cand_dep['name'], det_full_ec_version(cand_dep)) if path is None: full_mod_name = ActiveMNS().det_full_module_name( cand_dep) # no easyconfig found + no module available => missing dependency if not modtool.exist([full_mod_name])[0]: if cand_dep not in totally_missing: totally_missing.append(cand_dep) # no easyconfig found for dependency, but module is available # => add to list of missing easyconfigs elif cand_dep not in missing_easyconfigs: _log.debug( "Irresolvable dependency found (no easyconfig file): %s", cand_dep) missing_easyconfigs.append(cand_dep) # remove irresolvable dependency from list of dependencies so we can continue entry['dependencies'].remove(cand_dep) # add dummy entry for this dependency, so --dry-run for example can still report the dep additional.append({ 'dependencies': [], 'ec': None, 'full_mod_name': full_mod_name, 'spec': None, }) else: _log.info("Robot: resolving dependency %s with %s" % (cand_dep, path)) # build specs should not be passed down to resolved dependencies, # to avoid that e.g. --try-toolchain trickles down into the used toolchain itself hidden = cand_dep.get('hidden', False) processed_ecs = process_easyconfig( path, validate=not retain_all_deps, hidden=hidden) # ensure that selected easyconfig provides required dependency verify_easyconfig_filename(path, cand_dep, parsed_ec=processed_ecs) for ec in processed_ecs: if ec not in easyconfigs + additional: additional.append(ec) _log.debug("Added %s as dependency of %s" % (ec, entry)) else: mod_name = EasyBuildMNS().det_full_module_name(entry['ec']) _log.debug( "No more candidate dependencies to resolve for %s" % mod_name) # add additional (new) easyconfigs to list of stuff to process easyconfigs.extend(additional) _log.debug("Unprocessed dependencies: %s", easyconfigs) elif not robot: # no use in continuing if robot is not enabled, dependencies won't be resolved anyway missing_deps = [ dep for x in easyconfigs for dep in x['dependencies'] ] if missing_deps: raise_error_missing_deps( missing_deps, extra_msg="enable dependency resolution via --robot?") if totally_missing: raise_error_missing_deps( totally_missing, extra_msg="no easyconfig file or existing module found") if missing_easyconfigs: if raise_error_missing_ecs: raise_error_missing_deps( missing_easyconfigs, extra_msg="no easyconfig file found in robot search path") else: _log.warning("No easyconfig files found for: %s", missing_easyconfigs) _log.info("Dependency resolution complete, building as follows: %s", ordered_ecs) return ordered_ecs
def resolve_dependencies(unprocessed, build_specs=None, retain_all_deps=False): """ Work through the list of easyconfigs to determine an optimal order @param unprocessed: list of easyconfigs @param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...) @param retain_all_deps: boolean indicating whether all dependencies must be retained, regardless of availability; retain all deps when True, check matching build option when False """ robot = build_option("robot_path") # retain all dependencies if specified by either the resp. build option or the dedicated named argument retain_all_deps = build_option("retain_all_deps") or retain_all_deps if retain_all_deps: # assume that no modules are available when forced, to retain all dependencies avail_modules = [] _log.info("Forcing all dependencies to be retained.") else: # Get a list of all available modules (format: [(name, installversion), ...]) avail_modules = modules_tool().available() if len(avail_modules) == 0: _log.warning("No installed modules. Your MODULEPATH is probably incomplete: %s" % os.getenv("MODULEPATH")) ordered_ecs = [] # all available modules can be used for resolving dependencies except those that will be installed being_installed = [p["full_mod_name"] for p in unprocessed] avail_modules = [m for m in avail_modules if not m in being_installed] _log.debug("unprocessed before resolving deps: %s" % unprocessed) # resolve all dependencies, put a safeguard in place to avoid an infinite loop (shouldn't occur though) irresolvable = [] loopcnt = 0 maxloopcnt = 10000 while unprocessed: # make sure this stops, we really don't want to get stuck in an infinite loop loopcnt += 1 if loopcnt > maxloopcnt: raise EasyBuildError( "Maximum loop cnt %s reached, so quitting (unprocessed: %s, irresolvable: %s)", maxloopcnt, unprocessed, irresolvable, ) # first try resolving dependencies without using external dependencies last_processed_count = -1 while len(avail_modules) > last_processed_count: last_processed_count = len(avail_modules) res = find_resolved_modules(unprocessed, avail_modules, retain_all_deps=retain_all_deps) more_ecs, unprocessed, avail_modules = res for ec in more_ecs: if not ec["full_mod_name"] in [x["full_mod_name"] for x in ordered_ecs]: ordered_ecs.append(ec) # dependencies marked as external modules should be resolved via available modules at this point missing_external_modules = [ d["full_mod_name"] for ec in unprocessed for d in ec["dependencies"] if d.get("external_module", False) ] if missing_external_modules: raise EasyBuildError( "Missing modules for one or more dependencies marked as external modules: %s", missing_external_modules ) # robot: look for existing dependencies, add them if robot and unprocessed: # rely on EasyBuild module naming scheme when resolving dependencies, since we know that will # generate sensible module names that include the necessary information for the resolution to work # (name, version, toolchain, versionsuffix) being_installed = [EasyBuildMNS().det_full_module_name(p["ec"]) for p in unprocessed] additional = [] for entry in unprocessed: # do not choose an entry that is being installed in the current run # if they depend, you probably want to rebuild them using the new dependency deps = entry["dependencies"] candidates = [d for d in deps if not EasyBuildMNS().det_full_module_name(d) in being_installed] if candidates: cand_dep = candidates[0] # find easyconfig, might not find any _log.debug("Looking for easyconfig for %s" % str(cand_dep)) # note: robot_find_easyconfig may return None path = robot_find_easyconfig(cand_dep["name"], det_full_ec_version(cand_dep)) if path is None: # no easyconfig found for dependency, add to list of irresolvable dependencies if cand_dep not in irresolvable: _log.debug("Irresolvable dependency found: %s" % cand_dep) irresolvable.append(cand_dep) # remove irresolvable dependency from list of dependencies so we can continue entry["dependencies"].remove(cand_dep) else: _log.info("Robot: resolving dependency %s with %s" % (cand_dep, path)) # build specs should not be passed down to resolved dependencies, # to avoid that e.g. --try-toolchain trickles down into the used toolchain itself hidden = cand_dep.get("hidden", False) processed_ecs = process_easyconfig(path, validate=not retain_all_deps, hidden=hidden) # ensure that selected easyconfig provides required dependency mods = [spec["ec"].full_mod_name for spec in processed_ecs] dep_mod_name = ActiveMNS().det_full_module_name(cand_dep) if not dep_mod_name in mods: raise EasyBuildError( "easyconfig file %s does not contain module %s (mods: %s)", path, dep_mod_name, mods ) for ec in processed_ecs: if not ec in unprocessed + additional: additional.append(ec) _log.debug("Added %s as dependency of %s" % (ec, entry)) else: mod_name = EasyBuildMNS().det_full_module_name(entry["ec"]) _log.debug("No more candidate dependencies to resolve for %s" % mod_name) # add additional (new) easyconfigs to list of stuff to process unprocessed.extend(additional) _log.debug("Unprocessed dependencies: %s", unprocessed) elif not robot: # no use in continuing if robot is not enabled, dependencies won't be resolved anyway irresolvable = [dep for x in unprocessed for dep in x["dependencies"]] break if irresolvable: _log.warning("Irresolvable dependencies (details): %s" % irresolvable) irresolvable_mods_eb = [EasyBuildMNS().det_full_module_name(dep) for dep in irresolvable] _log.warning("Irresolvable dependencies (EasyBuild module names): %s" % ", ".join(irresolvable_mods_eb)) irresolvable_mods = [ActiveMNS().det_full_module_name(dep) for dep in irresolvable] raise EasyBuildError("Irresolvable dependencies encountered: %s", ", ".join(irresolvable_mods)) _log.info("Dependency resolution complete, building as follows: %s" % ordered_ecs) return ordered_ecs
def resolve_dependencies(unprocessed, build_specs=None, retain_all_deps=False): """ Work through the list of easyconfigs to determine an optimal order @param unprocessed: list of easyconfigs @param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...) """ robot = build_option('robot_path') retain_all_deps = build_option('retain_all_deps') or retain_all_deps if retain_all_deps: # assume that no modules are available when forced, to retain all dependencies avail_modules = [] _log.info("Forcing all dependencies to be retained.") else: # Get a list of all available modules (format: [(name, installversion), ...]) avail_modules = modules_tool().available() if len(avail_modules) == 0: _log.warning("No installed modules. Your MODULEPATH is probably incomplete: %s" % os.getenv('MODULEPATH')) ordered_ecs = [] # all available modules can be used for resolving dependencies except those that will be installed being_installed = [p['module'] for p in unprocessed] avail_modules = [m for m in avail_modules if not m in being_installed] _log.debug('unprocessed before resolving deps: %s' % unprocessed) # resolve all dependencies, put a safeguard in place to avoid an infinite loop (shouldn't occur though) irresolvable = [] loopcnt = 0 maxloopcnt = 10000 while unprocessed: # make sure this stops, we really don't want to get stuck in an infinite loop loopcnt += 1 if loopcnt > maxloopcnt: tup = (maxloopcnt, unprocessed, irresolvable) msg = "Maximum loop cnt %s reached, so quitting (unprocessed: %s, irresolvable: %s)" % tup _log.error(msg) # first try resolving dependencies without using external dependencies last_processed_count = -1 while len(avail_modules) > last_processed_count: last_processed_count = len(avail_modules) more_ecs, unprocessed, avail_modules = find_resolved_modules(unprocessed, avail_modules) for ec in more_ecs: if not ec['module'] in [x['module'] for x in ordered_ecs]: ordered_ecs.append(ec) # robot: look for existing dependencies, add them if robot and unprocessed: being_installed = [det_full_module_name(p['ec'], eb_ns=True) for p in unprocessed] additional = [] for i, entry in enumerate(unprocessed): # do not choose an entry that is being installed in the current run # if they depend, you probably want to rebuild them using the new dependency deps = entry['dependencies'] candidates = [d for d in deps if not det_full_module_name(d, eb_ns=True) in being_installed] if len(candidates) > 0: cand_dep = candidates[0] # find easyconfig, might not find any _log.debug("Looking for easyconfig for %s" % str(cand_dep)) # note: robot_find_easyconfig may return None path = robot_find_easyconfig(robot, cand_dep['name'], det_full_ec_version(cand_dep)) if path is None: # no easyconfig found for dependency, add to list of irresolvable dependencies if cand_dep not in irresolvable: _log.debug("Irresolvable dependency found: %s" % cand_dep) irresolvable.append(cand_dep) # remove irresolvable dependency from list of dependencies so we can continue entry['dependencies'].remove(cand_dep) else: _log.info("Robot: resolving dependency %s with %s" % (cand_dep, path)) # build specs should not be passed down to resolved dependencies, # to avoid that e.g. --try-toolchain trickles down into the used toolchain itself processed_ecs = process_easyconfig(path, validate=not retain_all_deps) # ensure that selected easyconfig provides required dependency mods = [det_full_module_name(spec['ec']) for spec in processed_ecs] dep_mod_name = det_full_module_name(cand_dep) if not dep_mod_name in mods: tup = (path, dep_mod_name, mods) _log.error("easyconfig file %s does not contain module %s (mods: %s)" % tup) for ec in processed_ecs: if not ec in unprocessed + additional: additional.append(ec) _log.debug("Added %s as dependency of %s" % (ec, entry)) else: mod_name = det_full_module_name(entry['ec'], eb_ns=True) _log.debug("No more candidate dependencies to resolve for %s" % mod_name) # add additional (new) easyconfigs to list of stuff to process unprocessed.extend(additional) elif not robot: # no use in continuing if robot is not enabled, dependencies won't be resolved anyway irresolvable = [dep for x in unprocessed for dep in x['dependencies']] break if irresolvable: irresolvable_mod_deps = [(det_full_module_name(dep, eb_ns=True), dep) for dep in irresolvable] _log.error('Irresolvable dependencies encountered: %s' % irresolvable_mod_deps) _log.info("Dependency resolution complete, building as follows:\n%s" % ordered_ecs) return ordered_ecs
def resolve_dependencies(easyconfigs, modtool, retain_all_deps=False, raise_error_missing_ecs=True): """ Work through the list of easyconfigs to determine an optimal order :param easyconfigs: list of easyconfigs :param modtool: ModulesTool instance to use :param retain_all_deps: boolean indicating whether all dependencies must be retained, regardless of availability; retain all deps when True, check matching build option when False :param raise_error_missing_ecs: raise an error when one or more easyconfig files could not be found """ robot = build_option('robot_path') # retain all dependencies if specified by either the resp. build option or the dedicated named argument retain_all_deps = build_option('retain_all_deps') or retain_all_deps avail_modules = modtool.available() if retain_all_deps: # assume that no modules are available when forced, to retain all dependencies avail_modules = [] _log.info("Forcing all dependencies to be retained.") else: if len(avail_modules) == 0: _log.warning("No installed modules. Your MODULEPATH is probably incomplete: %s" % os.getenv('MODULEPATH')) ordered_ecs = [] # all available modules can be used for resolving dependencies except those that will be installed being_installed = [p['full_mod_name'] for p in easyconfigs] avail_modules = [m for m in avail_modules if m not in being_installed] _log.debug('easyconfigs before resolving deps: %s', easyconfigs) totally_missing, missing_easyconfigs = [], [] # resolve all dependencies, put a safeguard in place to avoid an infinite loop (shouldn't occur though) loopcnt = 0 maxloopcnt = 10000 while easyconfigs: # make sure this stops, we really don't want to get stuck in an infinite loop loopcnt += 1 if loopcnt > maxloopcnt: raise EasyBuildError("Maximum loop cnt %s reached, so quitting (easyconfigs: %s, missing_easyconfigs: %s)", maxloopcnt, easyconfigs, missing_easyconfigs) # first try resolving dependencies without using external dependencies last_processed_count = -1 while len(avail_modules) > last_processed_count: last_processed_count = len(avail_modules) res = find_resolved_modules(easyconfigs, avail_modules, modtool, retain_all_deps=retain_all_deps) resolved_ecs, easyconfigs, avail_modules = res ordered_ec_mod_names = [x['full_mod_name'] for x in ordered_ecs] for ec in resolved_ecs: # only add easyconfig if it's not included yet (based on module name) if not ec['full_mod_name'] in ordered_ec_mod_names: ordered_ecs.append(ec) # dependencies marked as external modules should be resolved via available modules at this point missing_external_modules = [d['full_mod_name'] for ec in easyconfigs for d in ec['dependencies'] if d.get('external_module', False)] if missing_external_modules: raise EasyBuildError("Missing modules for dependencies marked as external modules: %s", ', '.join(missing_external_modules)) # robot: look for existing dependencies, add them if robot and easyconfigs: # rely on EasyBuild module naming scheme when resolving dependencies, since we know that will # generate sensible module names that include the necessary information for the resolution to work # (name, version, toolchain, versionsuffix) being_installed = [EasyBuildMNS().det_full_module_name(p['ec']) for p in easyconfigs] additional = [] for entry in easyconfigs: # do not choose an entry that is being installed in the current run # if they depend, you probably want to rebuild them using the new dependency deps = entry['dependencies'] candidates = [d for d in deps if not EasyBuildMNS().det_full_module_name(d) in being_installed] if candidates: cand_dep = candidates[0] # find easyconfig, might not find any _log.debug("Looking for easyconfig for %s" % str(cand_dep)) # note: robot_find_easyconfig may return None path = robot_find_easyconfig(cand_dep['name'], det_full_ec_version(cand_dep)) if path is None: full_mod_name = ActiveMNS().det_full_module_name(cand_dep) # no easyconfig found + no module available => missing dependency if not modtool.exist([full_mod_name])[0]: if cand_dep not in totally_missing: totally_missing.append(cand_dep) # no easyconfig found for dependency, but module is available # => add to list of missing easyconfigs elif cand_dep not in missing_easyconfigs: _log.debug("Irresolvable dependency found (no easyconfig file): %s", cand_dep) missing_easyconfigs.append(cand_dep) # remove irresolvable dependency from list of dependencies so we can continue entry['dependencies'].remove(cand_dep) # add dummy entry for this dependency, so --dry-run for example can still report the dep additional.append({ 'dependencies': [], 'ec': None, 'full_mod_name': full_mod_name, 'spec': None, }) else: _log.info("Robot: resolving dependency %s with %s" % (cand_dep, path)) # build specs should not be passed down to resolved dependencies, # to avoid that e.g. --try-toolchain trickles down into the used toolchain itself hidden = cand_dep.get('hidden', False) processed_ecs = process_easyconfig(path, validate=not retain_all_deps, hidden=hidden) # ensure that selected easyconfig provides required dependency verify_easyconfig_filename(path, cand_dep, parsed_ec=processed_ecs) for ec in processed_ecs: if ec not in easyconfigs + additional: additional.append(ec) _log.debug("Added %s as dependency of %s" % (ec, entry)) else: mod_name = EasyBuildMNS().det_full_module_name(entry['ec']) _log.debug("No more candidate dependencies to resolve for %s" % mod_name) # add additional (new) easyconfigs to list of stuff to process easyconfigs.extend(additional) _log.debug("Unprocessed dependencies: %s", easyconfigs) elif not robot: # no use in continuing if robot is not enabled, dependencies won't be resolved anyway missing_deps = [dep for x in easyconfigs for dep in x['dependencies']] if missing_deps: raise_error_missing_deps(missing_deps, extra_msg="enable dependency resolution via --robot?") if totally_missing: raise_error_missing_deps(totally_missing, extra_msg="no easyconfig file or existing module found") if missing_easyconfigs: if raise_error_missing_ecs: raise_error_missing_deps(missing_easyconfigs, extra_msg="no easyconfig file found in robot search path") else: _log.warning("No easyconfig files found for: %s", missing_easyconfigs) _log.info("Dependency resolution complete, building as follows: %s", ordered_ecs) return ordered_ecs
def get_toolchain_hierarchy(parent_toolchain): """ Determine list of subtoolchains for specified parent toolchain. Result starts with the most minimal subtoolchains first, ends with specified toolchain. The dummy toolchain is considered the most minimal subtoolchain only if the add_dummy_to_minimal_toolchains build option is enabled. @param parent_toolchain: dictionary with name/version of parent toolchain """ # obtain list of all possible subtoolchains _, all_tc_classes = search_toolchain("") subtoolchains = dict((tc_class.NAME, getattr(tc_class, "SUBTOOLCHAIN", None)) for tc_class in all_tc_classes) current_tc_name, current_tc_version = parent_toolchain["name"], parent_toolchain["version"] subtoolchain_name, subtoolchain_version = subtoolchains[current_tc_name], None # the parent toolchain is at the top of the hierarchy toolchain_hierarchy = [parent_toolchain] while subtoolchain_name: # grab the easyconfig of the current toolchain and search the dependencies for a version of the subtoolchain path = robot_find_easyconfig(current_tc_name, current_tc_version) if path is None: raise EasyBuildError( "Could not find easyconfig for %(name)s toolchain version %(version)s", current_tc_name, current_tc_version, ) # parse the easyconfig parsed_ec = process_easyconfig(path)[0] # search the dependencies for the version of the subtoolchain dep_tcs = [ dep_toolchain["toolchain"] for dep_toolchain in parsed_ec["dependencies"] if dep_toolchain["toolchain"]["name"] == subtoolchain_name ] unique_dep_tc_versions = set([dep_tc["version"] for dep_tc in dep_tcs]) if len(unique_dep_tc_versions) == 1: subtoolchain_version = dep_tcs[0]["version"] elif len(unique_dep_tc_versions) == 0: if subtoolchain_name == DUMMY_TOOLCHAIN_NAME: subtoolchain_version = "" else: raise EasyBuildError( "No version found for subtoolchain %s in dependencies of %s", subtoolchain_name, current_tc_name ) else: raise EasyBuildError( "Multiple versions of %s found in dependencies of toolchain %s: %s", subtoolchain_name, current_tc_name, unique_dep_tc_versions, ) if subtoolchain_name == DUMMY_TOOLCHAIN_NAME and not build_option("add_dummy_to_minimal_toolchains"): # we're done break # add to hierarchy and move to next current_tc_name, current_tc_version = subtoolchain_name, subtoolchain_version subtoolchain_name, subtoolchain_version = subtoolchains[current_tc_name], None toolchain_hierarchy.insert(0, {"name": current_tc_name, "version": current_tc_version}) _log.info("Found toolchain hierarchy for toolchain %s: %s", parent_toolchain, toolchain_hierarchy) return toolchain_hierarchy
def get_toolchain_hierarchy(parent_toolchain): """ Determine list of subtoolchains for specified parent toolchain. Result starts with the most minimal subtoolchains first, ends with specified toolchain. The dummy toolchain is considered the most minimal subtoolchain only if the add_dummy_to_minimal_toolchains build option is enabled. @param parent_toolchain: dictionary with name/version of parent toolchain """ # obtain list of all possible subtoolchains _, all_tc_classes = search_toolchain('') subtoolchains = dict( (tc_class.NAME, getattr(tc_class, 'SUBTOOLCHAIN', None)) for tc_class in all_tc_classes) current_tc_name, current_tc_version = parent_toolchain[ 'name'], parent_toolchain['version'] subtoolchain_name, subtoolchain_version = subtoolchains[ current_tc_name], None # the parent toolchain is at the top of the hierarchy toolchain_hierarchy = [parent_toolchain] while subtoolchain_name: # grab the easyconfig of the current toolchain and search the dependencies for a version of the subtoolchain path = robot_find_easyconfig(current_tc_name, current_tc_version) if path is None: raise EasyBuildError( "Could not find easyconfig for %(name)s toolchain version %(version)s", current_tc_name, current_tc_version) # parse the easyconfig parsed_ec = process_easyconfig(path)[0] # search the dependencies for the version of the subtoolchain dep_tcs = [ dep_toolchain['toolchain'] for dep_toolchain in parsed_ec['dependencies'] if dep_toolchain['toolchain']['name'] == subtoolchain_name ] unique_dep_tc_versions = set([dep_tc['version'] for dep_tc in dep_tcs]) if len(unique_dep_tc_versions) == 1: subtoolchain_version = dep_tcs[0]['version'] elif len(unique_dep_tc_versions) == 0: # only retain GCCcore as subtoolchain if version was found if subtoolchain_name == GCCcore.NAME: _log.info( "No version found for %s; assuming legacy toolchain and skipping it as subtoolchain.", subtoolchain_name) subtoolchain_name = GCCcore.SUBTOOLCHAIN subtoolchain_version = '' # dummy toolchain: end of the line elif subtoolchain_name == DUMMY_TOOLCHAIN_NAME: subtoolchain_version = '' else: raise EasyBuildError( "No version found for subtoolchain %s in dependencies of %s", subtoolchain_name, current_tc_name) else: raise EasyBuildError( "Multiple versions of %s found in dependencies of toolchain %s: %s", subtoolchain_name, current_tc_name, unique_dep_tc_versions) if subtoolchain_name == DUMMY_TOOLCHAIN_NAME and not build_option( 'add_dummy_to_minimal_toolchains'): # we're done break # add to hierarchy and move to next current_tc_name, current_tc_version = subtoolchain_name, subtoolchain_version subtoolchain_name, subtoolchain_version = subtoolchains[ current_tc_name], None toolchain_hierarchy.insert(0, { 'name': current_tc_name, 'version': current_tc_version }) _log.info("Found toolchain hierarchy for toolchain %s: %s", parent_toolchain, toolchain_hierarchy) return toolchain_hierarchy