def find_resolved_modules(unprocessed, avail_modules): """ Find easyconfigs in 1st argument which can be fully resolved using modules specified in 2nd argument """ ordered_ecs = [] new_avail_modules = avail_modules[:] new_unprocessed = [] for ec in unprocessed: new_ec = ec.copy() new_ec['dependencies'] = [d for d in new_ec['dependencies'] if not det_full_module_name(d) in new_avail_modules] if len(new_ec['dependencies']) == 0: _log.debug("Adding easyconfig %s to final list" % new_ec['spec']) ordered_ecs.append(new_ec) new_avail_modules.append(ec['module']) else: new_unprocessed.append(new_ec) return ordered_ecs, new_unprocessed, new_avail_modules
def print_dry_run(easyconfigs, short=False, build_specs=None): """ Print dry run information @param easyconfigs: list of easyconfig files @param short: print short output (use a variable for the common prefix) @param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...) """ lines = [] if build_option('robot_path') is None: lines.append("Dry run: printing build status of easyconfigs") all_specs = easyconfigs else: lines.append("Dry run: printing build status of easyconfigs and dependencies") all_specs = resolve_dependencies(easyconfigs, build_specs=build_specs, retain_all_deps=True) unbuilt_specs = skip_available(all_specs, testing=True) dry_run_fmt = " * [%1s] %s (module: %s)" # markdown compatible (list of items with checkboxes in front) var_name = 'CFGS' common_prefix = det_common_path_prefix([spec['spec'] for spec in all_specs]) # only allow short if common prefix is long enough short = short and common_prefix is not None and len(common_prefix) > len(var_name) * 2 for spec in all_specs: if spec in unbuilt_specs: ans = ' ' else: ans = 'x' mod = det_full_module_name(spec['ec']) if short: item = os.path.join('$%s' % var_name, spec['spec'][len(common_prefix) + 1:]) else: item = spec['spec'] lines.append(dry_run_fmt % (ans, item, mod)) if short: # insert after 'Dry run:' message lines.insert(1, "%s=%s" % (var_name, common_prefix)) silent = build_option('silent') print_msg('\n'.join(lines), log=_log, silent=silent, prefix=False)
def mk_node_name(spec): if omit_versions: return spec['name'] else: return det_full_module_name(spec)
def resolve_dependencies(unprocessed, build_specs=None, retain_all_deps=False): """ Work through the list of easyconfigs to determine an optimal order @param unprocessed: list of easyconfigs @param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...) """ robot = build_option('robot_path') retain_all_deps = build_option('retain_all_deps') or retain_all_deps if retain_all_deps: # assume that no modules are available when forced, to retain all dependencies avail_modules = [] _log.info("Forcing all dependencies to be retained.") else: # Get a list of all available modules (format: [(name, installversion), ...]) avail_modules = modules_tool().available() if len(avail_modules) == 0: _log.warning("No installed modules. Your MODULEPATH is probably incomplete: %s" % os.getenv('MODULEPATH')) ordered_ecs = [] # all available modules can be used for resolving dependencies except those that will be installed being_installed = [p['module'] for p in unprocessed] avail_modules = [m for m in avail_modules if not m in being_installed] _log.debug('unprocessed before resolving deps: %s' % unprocessed) # resolve all dependencies, put a safeguard in place to avoid an infinite loop (shouldn't occur though) irresolvable = [] loopcnt = 0 maxloopcnt = 10000 while unprocessed: # make sure this stops, we really don't want to get stuck in an infinite loop loopcnt += 1 if loopcnt > maxloopcnt: tup = (maxloopcnt, unprocessed, irresolvable) msg = "Maximum loop cnt %s reached, so quitting (unprocessed: %s, irresolvable: %s)" % tup _log.error(msg) # first try resolving dependencies without using external dependencies last_processed_count = -1 while len(avail_modules) > last_processed_count: last_processed_count = len(avail_modules) more_ecs, unprocessed, avail_modules = find_resolved_modules(unprocessed, avail_modules) for ec in more_ecs: if not ec['module'] in [x['module'] for x in ordered_ecs]: ordered_ecs.append(ec) # robot: look for existing dependencies, add them if robot and unprocessed: being_installed = [det_full_module_name(p['ec'], eb_ns=True) for p in unprocessed] additional = [] for i, entry in enumerate(unprocessed): # do not choose an entry that is being installed in the current run # if they depend, you probably want to rebuild them using the new dependency deps = entry['dependencies'] candidates = [d for d in deps if not det_full_module_name(d, eb_ns=True) in being_installed] if len(candidates) > 0: cand_dep = candidates[0] # find easyconfig, might not find any _log.debug("Looking for easyconfig for %s" % str(cand_dep)) # note: robot_find_easyconfig may return None path = robot_find_easyconfig(robot, cand_dep['name'], det_full_ec_version(cand_dep)) if path is None: # no easyconfig found for dependency, add to list of irresolvable dependencies if cand_dep not in irresolvable: _log.debug("Irresolvable dependency found: %s" % cand_dep) irresolvable.append(cand_dep) # remove irresolvable dependency from list of dependencies so we can continue entry['dependencies'].remove(cand_dep) else: _log.info("Robot: resolving dependency %s with %s" % (cand_dep, path)) # build specs should not be passed down to resolved dependencies, # to avoid that e.g. --try-toolchain trickles down into the used toolchain itself processed_ecs = process_easyconfig(path, validate=not retain_all_deps) # ensure that selected easyconfig provides required dependency mods = [det_full_module_name(spec['ec']) for spec in processed_ecs] dep_mod_name = det_full_module_name(cand_dep) if not dep_mod_name in mods: tup = (path, dep_mod_name, mods) _log.error("easyconfig file %s does not contain module %s (mods: %s)" % tup) for ec in processed_ecs: if not ec in unprocessed + additional: additional.append(ec) _log.debug("Added %s as dependency of %s" % (ec, entry)) else: mod_name = det_full_module_name(entry['ec'], eb_ns=True) _log.debug("No more candidate dependencies to resolve for %s" % mod_name) # add additional (new) easyconfigs to list of stuff to process unprocessed.extend(additional) elif not robot: # no use in continuing if robot is not enabled, dependencies won't be resolved anyway irresolvable = [dep for x in unprocessed for dep in x['dependencies']] break if irresolvable: irresolvable_mod_deps = [(det_full_module_name(dep, eb_ns=True), dep) for dep in irresolvable] _log.error('Irresolvable dependencies encountered: %s' % irresolvable_mod_deps) _log.info("Dependency resolution complete, building as follows:\n%s" % ordered_ecs) return ordered_ecs