def find_resolved_modules(unprocessed, avail_modules):
"""
Find easyconfigs in 1st argument which can be fully resolved using modules specified in 2nd argument
"""
ordered_ecs = []
new_avail_modules = avail_modules[:]
new_unprocessed = []
for ec in unprocessed:
new_ec = ec.copy()
new_ec['dependencies'] = [d for d in new_ec['dependencies'] if not det_full_module_name(d) in new_avail_modules]
if len(new_ec['dependencies']) == 0:
_log.debug("Adding easyconfig %s to final list" % new_ec['spec'])
ordered_ecs.append(new_ec)
new_avail_modules.append(ec['module'])
else:
new_unprocessed.append(new_ec)
return ordered_ecs, new_unprocessed, new_avail_modules
def print_dry_run(easyconfigs, short=False, build_specs=None):
"""
Print dry run information
@param easyconfigs: list of easyconfig files
@param short: print short output (use a variable for the common prefix)
@param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...)
"""
lines = []
if build_option('robot_path') is None:
lines.append("Dry run: printing build status of easyconfigs")
all_specs = easyconfigs
else:
lines.append("Dry run: printing build status of easyconfigs and dependencies")
all_specs = resolve_dependencies(easyconfigs, build_specs=build_specs, retain_all_deps=True)
unbuilt_specs = skip_available(all_specs, testing=True)
dry_run_fmt = " * [%1s] %s (module: %s)" # markdown compatible (list of items with checkboxes in front)
var_name = 'CFGS'
common_prefix = det_common_path_prefix([spec['spec'] for spec in all_specs])
# only allow short if common prefix is long enough
short = short and common_prefix is not None and len(common_prefix) > len(var_name) * 2
for spec in all_specs:
if spec in unbuilt_specs:
ans = ' '
else:
ans = 'x'
mod = det_full_module_name(spec['ec'])
if short:
item = os.path.join('$%s' % var_name, spec['spec'][len(common_prefix) + 1:])
else:
item = spec['spec']
lines.append(dry_run_fmt % (ans, item, mod))
if short:
# insert after 'Dry run:' message
lines.insert(1, "%s=%s" % (var_name, common_prefix))
silent = build_option('silent')
print_msg('\n'.join(lines), log=_log, silent=silent, prefix=False)
def mk_node_name(spec):
if omit_versions:
return spec['name']
else:
return det_full_module_name(spec)
def resolve_dependencies(unprocessed, build_specs=None, retain_all_deps=False):
"""
Work through the list of easyconfigs to determine an optimal order
@param unprocessed: list of easyconfigs
@param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...)
"""
robot = build_option('robot_path')
retain_all_deps = build_option('retain_all_deps') or retain_all_deps
if retain_all_deps:
# assume that no modules are available when forced, to retain all dependencies
avail_modules = []
_log.info("Forcing all dependencies to be retained.")
else:
# Get a list of all available modules (format: [(name, installversion), ...])
avail_modules = modules_tool().available()
if len(avail_modules) == 0:
_log.warning("No installed modules. Your MODULEPATH is probably incomplete: %s" % os.getenv('MODULEPATH'))
ordered_ecs = []
# all available modules can be used for resolving dependencies except those that will be installed
being_installed = [p['module'] for p in unprocessed]
avail_modules = [m for m in avail_modules if not m in being_installed]
_log.debug('unprocessed before resolving deps: %s' % unprocessed)
# resolve all dependencies, put a safeguard in place to avoid an infinite loop (shouldn't occur though)
irresolvable = []
loopcnt = 0
maxloopcnt = 10000
while unprocessed:
# make sure this stops, we really don't want to get stuck in an infinite loop
loopcnt += 1
if loopcnt > maxloopcnt:
tup = (maxloopcnt, unprocessed, irresolvable)
msg = "Maximum loop cnt %s reached, so quitting (unprocessed: %s, irresolvable: %s)" % tup
_log.error(msg)
# first try resolving dependencies without using external dependencies
last_processed_count = -1
while len(avail_modules) > last_processed_count:
last_processed_count = len(avail_modules)
more_ecs, unprocessed, avail_modules = find_resolved_modules(unprocessed, avail_modules)
for ec in more_ecs:
if not ec['module'] in [x['module'] for x in ordered_ecs]:
ordered_ecs.append(ec)
# robot: look for existing dependencies, add them
if robot and unprocessed:
being_installed = [det_full_module_name(p['ec'], eb_ns=True) for p in unprocessed]
additional = []
for i, entry in enumerate(unprocessed):
# do not choose an entry that is being installed in the current run
# if they depend, you probably want to rebuild them using the new dependency
deps = entry['dependencies']
candidates = [d for d in deps if not det_full_module_name(d, eb_ns=True) in being_installed]
if len(candidates) > 0:
cand_dep = candidates[0]
# find easyconfig, might not find any
_log.debug("Looking for easyconfig for %s" % str(cand_dep))
# note: robot_find_easyconfig may return None
path = robot_find_easyconfig(robot, cand_dep['name'], det_full_ec_version(cand_dep))
if path is None:
# no easyconfig found for dependency, add to list of irresolvable dependencies
if cand_dep not in irresolvable:
_log.debug("Irresolvable dependency found: %s" % cand_dep)
irresolvable.append(cand_dep)
# remove irresolvable dependency from list of dependencies so we can continue
entry['dependencies'].remove(cand_dep)
else:
_log.info("Robot: resolving dependency %s with %s" % (cand_dep, path))
# build specs should not be passed down to resolved dependencies,
# to avoid that e.g. --try-toolchain trickles down into the used toolchain itself
processed_ecs = process_easyconfig(path, validate=not retain_all_deps)
# ensure that selected easyconfig provides required dependency
mods = [det_full_module_name(spec['ec']) for spec in processed_ecs]
dep_mod_name = det_full_module_name(cand_dep)
if not dep_mod_name in mods:
tup = (path, dep_mod_name, mods)
_log.error("easyconfig file %s does not contain module %s (mods: %s)" % tup)
for ec in processed_ecs:
if not ec in unprocessed + additional:
additional.append(ec)
_log.debug("Added %s as dependency of %s" % (ec, entry))
else:
mod_name = det_full_module_name(entry['ec'], eb_ns=True)
_log.debug("No more candidate dependencies to resolve for %s" % mod_name)
# add additional (new) easyconfigs to list of stuff to process
unprocessed.extend(additional)
elif not robot:
# no use in continuing if robot is not enabled, dependencies won't be resolved anyway
irresolvable = [dep for x in unprocessed for dep in x['dependencies']]
break
if irresolvable:
irresolvable_mod_deps = [(det_full_module_name(dep, eb_ns=True), dep) for dep in irresolvable]
_log.error('Irresolvable dependencies encountered: %s' % irresolvable_mod_deps)
_log.info("Dependency resolution complete, building as follows:\n%s" % ordered_ecs)
return ordered_ecs
