def resolve_template_data(self): """Return template data for container recipe.""" os_deps = det_os_deps(self.easyconfigs) ec = self.easyconfigs[-1]['ec'] # We are using the default MNS inside the container docker_mns = EasyBuildMNS() init_modulepath = os.path.join("/app/modules/all", *docker_mns.det_init_modulepaths(ec)) mod_names = [ docker_mns.det_full_module_name(e['ec']) for e in self.easyconfigs ] eb_opts = [os.path.basename(e['spec']) for e in self.easyconfigs] return { 'os_deps': ' '.join(os_deps), 'eb_opts': ' '.join(eb_opts), 'init_modulepath': init_modulepath, 'mod_names': ' '.join(mod_names), }
def raise_error_missing_deps(missing_deps, extra_msg=None): """Raise error to report missing dependencies.""" _log.warning("Missing dependencies (details): %s", missing_deps) mod_names_eb = ', '.join(EasyBuildMNS().det_full_module_name(dep) for dep in missing_deps) _log.warning("Missing dependencies (EasyBuild module names): %s", mod_names_eb) mod_names = ', '.join(ActiveMNS().det_full_module_name(dep) for dep in missing_deps) error_msg = "Missing dependencies: %s" % mod_names if extra_msg: error_msg += ' (%s)' % extra_msg raise EasyBuildError(error_msg)
def resolve_dependencies(unprocessed, build_specs=None, retain_all_deps=False): """ Work through the list of easyconfigs to determine an optimal order @param unprocessed: list of easyconfigs @param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...) @param retain_all_deps: boolean indicating whether all dependencies must be retained, regardless of availability; retain all deps when True, check matching build option when False """ robot = build_option('robot_path') # retain all dependencies if specified by either the resp. build option or the dedicated named argument retain_all_deps = build_option('retain_all_deps') or retain_all_deps if retain_all_deps: # assume that no modules are available when forced, to retain all dependencies avail_modules = [] _log.info("Forcing all dependencies to be retained.") else: # Get a list of all available modules (format: [(name, installversion), ...]) avail_modules = modules_tool().available() if len(avail_modules) == 0: _log.warning( "No installed modules. Your MODULEPATH is probably incomplete: %s" % os.getenv('MODULEPATH')) ordered_ecs = [] # all available modules can be used for resolving dependencies except those that will be installed being_installed = [p['full_mod_name'] for p in unprocessed] avail_modules = [m for m in avail_modules if not m in being_installed] _log.debug('unprocessed before resolving deps: %s' % unprocessed) # resolve all dependencies, put a safeguard in place to avoid an infinite loop (shouldn't occur though) irresolvable = [] loopcnt = 0 maxloopcnt = 10000 while unprocessed: # make sure this stops, we really don't want to get stuck in an infinite loop loopcnt += 1 if loopcnt > maxloopcnt: tup = (maxloopcnt, unprocessed, irresolvable) msg = "Maximum loop cnt %s reached, so quitting (unprocessed: %s, irresolvable: %s)" % tup _log.error(msg) # first try resolving dependencies without using external dependencies last_processed_count = -1 while len(avail_modules) > last_processed_count: last_processed_count = len(avail_modules) res = find_resolved_modules(unprocessed, avail_modules, retain_all_deps=retain_all_deps) more_ecs, unprocessed, avail_modules = res for ec in more_ecs: if not ec['full_mod_name'] in [ x['full_mod_name'] for x in ordered_ecs ]: ordered_ecs.append(ec) # robot: look for existing dependencies, add them if robot and unprocessed: # rely on EasyBuild module naming scheme when resolving dependencies, since we know that will # generate sensible module names that include the necessary information for the resolution to work # (name, version, toolchain, versionsuffix) being_installed = [ EasyBuildMNS().det_full_module_name(p['ec']) for p in unprocessed ] additional = [] for entry in unprocessed: # do not choose an entry that is being installed in the current run # if they depend, you probably want to rebuild them using the new dependency deps = entry['dependencies'] candidates = [ d for d in deps if not EasyBuildMNS().det_full_module_name( d) in being_installed ] if candidates: cand_dep = candidates[0] # find easyconfig, might not find any _log.debug("Looking for easyconfig for %s" % str(cand_dep)) # note: robot_find_easyconfig may return None path = robot_find_easyconfig(cand_dep['name'], det_full_ec_version(cand_dep)) if path is None: # no easyconfig found for dependency, add to list of irresolvable dependencies if cand_dep not in irresolvable: _log.debug("Irresolvable dependency found: %s" % cand_dep) irresolvable.append(cand_dep) # remove irresolvable dependency from list of dependencies so we can continue entry['dependencies'].remove(cand_dep) else: _log.info("Robot: resolving dependency %s with %s" % (cand_dep, path)) # build specs should not be passed down to resolved dependencies, # to avoid that e.g. --try-toolchain trickles down into the used toolchain itself hidden = cand_dep.get('hidden', False) processed_ecs = process_easyconfig( path, validate=not retain_all_deps, hidden=hidden) # ensure that selected easyconfig provides required dependency mods = [ spec['ec'].full_mod_name for spec in processed_ecs ] dep_mod_name = ActiveMNS().det_full_module_name( cand_dep) if not dep_mod_name in mods: tup = (path, dep_mod_name, mods) _log.error( "easyconfig file %s does not contain module %s (mods: %s)" % tup) for ec in processed_ecs: if not ec in unprocessed + additional: additional.append(ec) _log.debug("Added %s as dependency of %s" % (ec, entry)) else: mod_name = EasyBuildMNS().det_full_module_name(entry['ec']) _log.debug( "No more candidate dependencies to resolve for %s" % mod_name) # add additional (new) easyconfigs to list of stuff to process unprocessed.extend(additional) elif not robot: # no use in continuing if robot is not enabled, dependencies won't be resolved anyway irresolvable = [ dep for x in unprocessed for dep in x['dependencies'] ] break if irresolvable: _log.warning("Irresolvable dependencies (details): %s" % irresolvable) irresolvable_mods_eb = [ EasyBuildMNS().det_full_module_name(dep) for dep in irresolvable ] _log.warning("Irresolvable dependencies (EasyBuild module names): %s" % ', '.join(irresolvable_mods_eb)) irresolvable_mods = [ ActiveMNS().det_full_module_name(dep) for dep in irresolvable ] _log.error('Irresolvable dependencies encountered: %s' % ', '.join(irresolvable_mods)) _log.info("Dependency resolution complete, building as follows:\n%s" % ordered_ecs) return ordered_ecs
def mk_dep_mod_name(spec): return tuple(EasyBuildMNS().det_full_module_name(spec).split( os.path.sep))
def det_install_subdir(self, ec): """Determine name of software installation subdirectory of install path, using EasyBuild MNS.""" return EasyBuildMNS.det_full_module_name(self, ec)
def resolve_dependencies(easyconfigs, modtool, retain_all_deps=False, raise_error_missing_ecs=True): """ Work through the list of easyconfigs to determine an optimal order :param easyconfigs: list of easyconfigs :param modtool: ModulesTool instance to use :param retain_all_deps: boolean indicating whether all dependencies must be retained, regardless of availability; retain all deps when True, check matching build option when False :param raise_error_missing_ecs: raise an error when one or more easyconfig files could not be found """ robot = build_option('robot_path') # retain all dependencies if specified by either the resp. build option or the dedicated named argument retain_all_deps = build_option('retain_all_deps') or retain_all_deps avail_modules = modtool.available() if retain_all_deps: # assume that no modules are available when forced, to retain all dependencies avail_modules = [] _log.info("Forcing all dependencies to be retained.") else: if len(avail_modules) == 0: _log.warning( "No installed modules. Your MODULEPATH is probably incomplete: %s" % os.getenv('MODULEPATH')) ordered_ecs = [] # all available modules can be used for resolving dependencies except those that will be installed being_installed = [p['full_mod_name'] for p in easyconfigs] avail_modules = [m for m in avail_modules if m not in being_installed] _log.debug('easyconfigs before resolving deps: %s', easyconfigs) totally_missing, missing_easyconfigs = [], [] # resolve all dependencies, put a safeguard in place to avoid an infinite loop (shouldn't occur though) loopcnt = 0 maxloopcnt = 10000 while easyconfigs: # make sure this stops, we really don't want to get stuck in an infinite loop loopcnt += 1 if loopcnt > maxloopcnt: raise EasyBuildError( "Maximum loop cnt %s reached, so quitting (easyconfigs: %s, missing_easyconfigs: %s)", maxloopcnt, easyconfigs, missing_easyconfigs) # first try resolving dependencies without using external dependencies last_processed_count = -1 while len(avail_modules) > last_processed_count: last_processed_count = len(avail_modules) res = find_resolved_modules(easyconfigs, avail_modules, modtool, retain_all_deps=retain_all_deps) resolved_ecs, easyconfigs, avail_modules = res ordered_ec_mod_names = [x['full_mod_name'] for x in ordered_ecs] for ec in resolved_ecs: # only add easyconfig if it's not included yet (based on module name) if not ec['full_mod_name'] in ordered_ec_mod_names: ordered_ecs.append(ec) # dependencies marked as external modules should be resolved via available modules at this point missing_external_modules = [ d['full_mod_name'] for ec in easyconfigs for d in ec['dependencies'] if d.get('external_module', False) ] if missing_external_modules: raise EasyBuildError( "Missing modules for dependencies marked as external modules: %s", ', '.join(missing_external_modules)) # robot: look for existing dependencies, add them if robot and easyconfigs: # rely on EasyBuild module naming scheme when resolving dependencies, since we know that will # generate sensible module names that include the necessary information for the resolution to work # (name, version, toolchain, versionsuffix) being_installed = [ EasyBuildMNS().det_full_module_name(p['ec']) for p in easyconfigs ] additional = [] for entry in easyconfigs: # do not choose an entry that is being installed in the current run # if they depend, you probably want to rebuild them using the new dependency deps = entry['dependencies'] candidates = [ d for d in deps if not EasyBuildMNS().det_full_module_name( d) in being_installed ] if candidates: cand_dep = candidates[0] # find easyconfig, might not find any _log.debug("Looking for easyconfig for %s" % str(cand_dep)) # note: robot_find_easyconfig may return None path = robot_find_easyconfig(cand_dep['name'], det_full_ec_version(cand_dep)) if path is None: full_mod_name = ActiveMNS().det_full_module_name( cand_dep) # no easyconfig found + no module available => missing dependency if not modtool.exist([full_mod_name])[0]: if cand_dep not in totally_missing: totally_missing.append(cand_dep) # no easyconfig found for dependency, but module is available # => add to list of missing easyconfigs elif cand_dep not in missing_easyconfigs: _log.debug( "Irresolvable dependency found (no easyconfig file): %s", cand_dep) missing_easyconfigs.append(cand_dep) # remove irresolvable dependency from list of dependencies so we can continue entry['dependencies'].remove(cand_dep) # add dummy entry for this dependency, so --dry-run for example can still report the dep additional.append({ 'dependencies': [], 'ec': None, 'full_mod_name': full_mod_name, 'spec': None, }) else: _log.info("Robot: resolving dependency %s with %s" % (cand_dep, path)) # build specs should not be passed down to resolved dependencies, # to avoid that e.g. --try-toolchain trickles down into the used toolchain itself hidden = cand_dep.get('hidden', False) processed_ecs = process_easyconfig( path, validate=not retain_all_deps, hidden=hidden) # ensure that selected easyconfig provides required dependency verify_easyconfig_filename(path, cand_dep, parsed_ec=processed_ecs) for ec in processed_ecs: if ec not in easyconfigs + additional: additional.append(ec) _log.debug("Added %s as dependency of %s" % (ec, entry)) else: mod_name = EasyBuildMNS().det_full_module_name(entry['ec']) _log.debug( "No more candidate dependencies to resolve for %s" % mod_name) # add additional (new) easyconfigs to list of stuff to process easyconfigs.extend(additional) _log.debug("Unprocessed dependencies: %s", easyconfigs) elif not robot: # no use in continuing if robot is not enabled, dependencies won't be resolved anyway missing_deps = [ dep for x in easyconfigs for dep in x['dependencies'] ] if missing_deps: raise_error_missing_deps( missing_deps, extra_msg="enable dependency resolution via --robot?") if totally_missing: raise_error_missing_deps( totally_missing, extra_msg="no easyconfig file or existing module found") if missing_easyconfigs: if raise_error_missing_ecs: raise_error_missing_deps( missing_easyconfigs, extra_msg="no easyconfig file found in robot search path") else: _log.warning("No easyconfig files found for: %s", missing_easyconfigs) _log.info("Dependency resolution complete, building as follows: %s", ordered_ecs) return ordered_ecs