def test_find_minimally_resolved_modules(self): """Test find_minimally_resolved_modules function.""" # replace log.experimental with log.warning to allow experimental code easybuild.framework.easyconfig.tools._log.experimental = easybuild.framework.easyconfig.tools._log.warning test_easyconfigs = os.path.join( os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') init_config( build_options={ 'valid_module_classes': module_classes(), 'robot_path': test_easyconfigs, }) barec = os.path.join(self.test_prefix, 'bar-1.2.3-goolf-1.4.10.eb') barec_txt = '\n'.join([ "easyblock = 'ConfigureMake'", "name = 'bar'", "version = '1.2.3'", "homepage = 'http://example.com'", "description = 'foo'", "toolchain = {'name': 'goolf', 'version': '1.4.10'}", # deliberately listing components of toolchain as dependencies without specifying subtoolchains, # to test resolving of dependencies with minimal toolchain # for each of these, we know test easyconfigs are available (which are required here) "dependencies = [", " ('OpenMPI', '1.6.4'),", # available with GCC/4.7.2 " ('OpenBLAS', '0.2.6', '-LAPACK-3.4.2'),", # available with gompi/1.4.10 " ('ScaLAPACK', '2.0.2', '-OpenBLAS-0.2.6-LAPACK-3.4.2'),", # available with gompi/1.4.10 " ('SQLite', '3.8.10.2'),", # only available with goolf/1.4.10 "]", ]) write_file(barec, barec_txt) bar = process_easyconfig(barec)[0] ecs = [bar] mods = [ 'gompi/1.4.10', 'goolf/1.4.10', # include modules for dependencies, with subtoolchains rather than full toolchain (except for SQLite) 'OpenMPI/1.6.4-GCC-4.7.2', 'OpenBLAS/0.2.6-gompi-1.4.10-LAPACK-3.4.2', 'ScaLAPACK/2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2', 'SQLite/3.8.10.2-GCC-4.7.2', ] ordered_ecs, new_easyconfigs, new_avail_modules = find_minimally_resolved_modules( ecs, mods, []) # all dependencies are resolved for easyconfigs included in ordered_ecs self.assertEqual(len(ordered_ecs), 1) self.assertEqual(ordered_ecs[0]['dependencies'], []) # module is added to list of available modules self.assertTrue(bar['ec'].full_mod_name in new_avail_modules) # nothing left self.assertEqual(new_easyconfigs, [])
def test_find_minimally_resolved_modules(self): """Test find_minimally_resolved_modules function.""" # replace log.experimental with log.warning to allow experimental code easybuild.framework.easyconfig.tools._log.experimental = easybuild.framework.easyconfig.tools._log.warning test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') init_config(build_options={ 'valid_module_classes': module_classes(), 'robot_path': test_easyconfigs, }) barec = os.path.join(self.test_prefix, 'bar-1.2.3-goolf-1.4.10.eb') barec_txt = '\n'.join([ "easyblock = 'ConfigureMake'", "name = 'bar'", "version = '1.2.3'", "homepage = 'http://example.com'", "description = 'foo'", "toolchain = {'name': 'goolf', 'version': '1.4.10'}", # deliberately listing components of toolchain as dependencies without specifying subtoolchains, # to test resolving of dependencies with minimal toolchain # for each of these, we know test easyconfigs are available (which are required here) "dependencies = [", " ('OpenMPI', '1.6.4'),", # available with GCC/4.7.2 " ('OpenBLAS', '0.2.6', '-LAPACK-3.4.2'),", # available with gompi/1.4.10 " ('ScaLAPACK', '2.0.2', '-OpenBLAS-0.2.6-LAPACK-3.4.2'),", # available with gompi/1.4.10 " ('SQLite', '3.8.10.2'),", # only available with goolf/1.4.10 "]", ]) write_file(barec, barec_txt) bar = process_easyconfig(barec)[0] ecs = [bar] mods = [ 'gompi/1.4.10', 'goolf/1.4.10', # include modules for dependencies, with subtoolchains rather than full toolchain (except for SQLite) 'OpenMPI/1.6.4-GCC-4.7.2', 'OpenBLAS/0.2.6-gompi-1.4.10-LAPACK-3.4.2', 'ScaLAPACK/2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2', 'SQLite/3.8.10.2-GCC-4.7.2', ] ordered_ecs, new_easyconfigs, new_avail_modules = find_minimally_resolved_modules(ecs, mods, []) # all dependencies are resolved for easyconfigs included in ordered_ecs self.assertEqual(len(ordered_ecs), 1) self.assertEqual(ordered_ecs[0]['dependencies'], []) # module is added to list of available modules self.assertTrue(bar['ec'].full_mod_name in new_avail_modules) # nothing left self.assertEqual(new_easyconfigs, [])
def resolve_dependencies(easyconfigs, retain_all_deps=False, minimal_toolchains=False, use_existing_modules=False): """ Work through the list of easyconfigs to determine an optimal order @param easyconfigs: list of easyconfigs @param retain_all_deps: boolean indicating whether all dependencies must be retained, regardless of availability; retain all deps when True, check matching build option when False @param minimal_toolchains: boolean for whether to try to resolve dependencies with minimum possible toolchain @param use_existing_modules: boolean for whether to prioritise the reuse of existing modules (works in combination with minimal_toolchains) """ robot = build_option('robot_path') # retain all dependencies if specified by either the resp. build option or the dedicated named argument retain_all_deps = build_option('retain_all_deps') or retain_all_deps existing_modules = modules_tool().available() if retain_all_deps: # assume that no modules are available when forced, to retain all dependencies avail_modules = [] _log.info("Forcing all dependencies to be retained.") else: # Get a list of all available modules (format: [(name, installversion), ...]) avail_modules = existing_modules[:] if len(avail_modules) == 0: _log.warning( "No installed modules. Your MODULEPATH is probably incomplete: %s" % os.getenv('MODULEPATH')) ordered_ecs = [] # all available modules can be used for resolving dependencies except those that will be installed being_installed = [p['full_mod_name'] for p in easyconfigs] avail_modules = [m for m in avail_modules if not m in being_installed] _log.debug('easyconfigs before resolving deps: %s' % easyconfigs) # resolve all dependencies, put a safeguard in place to avoid an infinite loop (shouldn't occur though) irresolvable = [] loopcnt = 0 maxloopcnt = 10000 while easyconfigs: # make sure this stops, we really don't want to get stuck in an infinite loop loopcnt += 1 if loopcnt > maxloopcnt: raise EasyBuildError( "Maximum loop cnt %s reached, so quitting (easyconfigs: %s, irresolvable: %s)", maxloopcnt, easyconfigs, irresolvable) # first try resolving dependencies without using external dependencies last_processed_count = -1 while len(avail_modules) > last_processed_count: last_processed_count = len(avail_modules) if minimal_toolchains: res = find_minimally_resolved_modules( easyconfigs, avail_modules, existing_modules, retain_all_deps=retain_all_deps, use_existing_modules=use_existing_modules) else: res = find_resolved_modules(easyconfigs, avail_modules, retain_all_deps=retain_all_deps) more_ecs, easyconfigs, avail_modules = res for ec in more_ecs: if not ec['full_mod_name'] in [ x['full_mod_name'] for x in ordered_ecs ]: ordered_ecs.append(ec) # dependencies marked as external modules should be resolved via available modules at this point missing_external_modules = [ d['full_mod_name'] for ec in easyconfigs for d in ec['dependencies'] if d.get('external_module', False) ] if missing_external_modules: raise EasyBuildError( "Missing modules for one or more dependencies marked as external modules: %s", missing_external_modules) # robot: look for existing dependencies, add them if robot and easyconfigs: # rely on EasyBuild module naming scheme when resolving dependencies, since we know that will # generate sensible module names that include the necessary information for the resolution to work # (name, version, toolchain, versionsuffix) being_installed = [ EasyBuildMNS().det_full_module_name(p['ec']) for p in easyconfigs ] additional = [] for entry in easyconfigs: # do not choose an entry that is being installed in the current run # if they depend, you probably want to rebuild them using the new dependency deps = entry['dependencies'] candidates = [ d for d in deps if not EasyBuildMNS().det_full_module_name( d) in being_installed ] if candidates: cand_dep = candidates[0] # find easyconfig, might not find any _log.debug("Looking for easyconfig for %s" % str(cand_dep)) # note: robot_find_easyconfig may return None path = robot_find_easyconfig(cand_dep['name'], det_full_ec_version(cand_dep)) if path is None: # no easyconfig found for dependency, add to list of irresolvable dependencies if cand_dep not in irresolvable: _log.debug("Irresolvable dependency found: %s" % cand_dep) irresolvable.append(cand_dep) # remove irresolvable dependency from list of dependencies so we can continue entry['dependencies'].remove(cand_dep) else: _log.info("Robot: resolving dependency %s with %s" % (cand_dep, path)) # build specs should not be passed down to resolved dependencies, # to avoid that e.g. --try-toolchain trickles down into the used toolchain itself hidden = cand_dep.get('hidden', False) processed_ecs = process_easyconfig( path, validate=not retain_all_deps, hidden=hidden) # ensure that selected easyconfig provides required dependency mods = [ spec['ec'].full_mod_name for spec in processed_ecs ] dep_mod_name = ActiveMNS().det_full_module_name( cand_dep) if not dep_mod_name in mods: raise EasyBuildError( "easyconfig file %s does not contain module %s (mods: %s)", path, dep_mod_name, mods) for ec in processed_ecs: if not ec in easyconfigs + additional: additional.append(ec) _log.debug("Added %s as dependency of %s" % (ec, entry)) else: mod_name = EasyBuildMNS().det_full_module_name(entry['ec']) _log.debug( "No more candidate dependencies to resolve for %s" % mod_name) # add additional (new) easyconfigs to list of stuff to process easyconfigs.extend(additional) _log.debug("Unprocessed dependencies: %s", easyconfigs) elif not robot: # no use in continuing if robot is not enabled, dependencies won't be resolved anyway irresolvable = [ dep for x in easyconfigs for dep in x['dependencies'] ] break if irresolvable: _log.warning("Irresolvable dependencies (details): %s" % irresolvable) irresolvable_mods_eb = [ EasyBuildMNS().det_full_module_name(dep) for dep in irresolvable ] _log.warning("Irresolvable dependencies (EasyBuild module names): %s" % ', '.join(irresolvable_mods_eb)) irresolvable_mods = [ ActiveMNS().det_full_module_name(dep) for dep in irresolvable ] raise EasyBuildError("Irresolvable dependencies encountered: %s", ', '.join(irresolvable_mods)) _log.info("Dependency resolution complete, building as follows: %s" % ordered_ecs) return ordered_ecs
def resolve_dependencies(easyconfigs, retain_all_deps=False, minimal_toolchains=False, use_existing_modules=False): """ Work through the list of easyconfigs to determine an optimal order @param easyconfigs: list of easyconfigs @param retain_all_deps: boolean indicating whether all dependencies must be retained, regardless of availability; retain all deps when True, check matching build option when False @param minimal_toolchains: boolean for whether to try to resolve dependencies with minimum possible toolchain @param use_existing_modules: boolean for whether to prioritise the reuse of existing modules (works in combination with minimal_toolchains) """ robot = build_option('robot_path') # retain all dependencies if specified by either the resp. build option or the dedicated named argument retain_all_deps = build_option('retain_all_deps') or retain_all_deps existing_modules = modules_tool().available() if retain_all_deps: # assume that no modules are available when forced, to retain all dependencies avail_modules = [] _log.info("Forcing all dependencies to be retained.") else: # Get a list of all available modules (format: [(name, installversion), ...]) avail_modules = existing_modules[:] if len(avail_modules) == 0: _log.warning("No installed modules. Your MODULEPATH is probably incomplete: %s" % os.getenv('MODULEPATH')) ordered_ecs = [] # all available modules can be used for resolving dependencies except those that will be installed being_installed = [p['full_mod_name'] for p in easyconfigs] avail_modules = [m for m in avail_modules if not m in being_installed] _log.debug('easyconfigs before resolving deps: %s' % easyconfigs) # resolve all dependencies, put a safeguard in place to avoid an infinite loop (shouldn't occur though) irresolvable = [] loopcnt = 0 maxloopcnt = 10000 while easyconfigs: # make sure this stops, we really don't want to get stuck in an infinite loop loopcnt += 1 if loopcnt > maxloopcnt: raise EasyBuildError("Maximum loop cnt %s reached, so quitting (easyconfigs: %s, irresolvable: %s)", maxloopcnt, easyconfigs, irresolvable) # first try resolving dependencies without using external dependencies last_processed_count = -1 while len(avail_modules) > last_processed_count: last_processed_count = len(avail_modules) if minimal_toolchains: res = find_minimally_resolved_modules(easyconfigs, avail_modules, existing_modules, retain_all_deps=retain_all_deps, use_existing_modules=use_existing_modules) else: res = find_resolved_modules(easyconfigs, avail_modules, retain_all_deps=retain_all_deps) more_ecs, easyconfigs, avail_modules = res for ec in more_ecs: if not ec['full_mod_name'] in [x['full_mod_name'] for x in ordered_ecs]: ordered_ecs.append(ec) # dependencies marked as external modules should be resolved via available modules at this point missing_external_modules = [d['full_mod_name'] for ec in easyconfigs for d in ec['dependencies'] if d.get('external_module', False)] if missing_external_modules: raise EasyBuildError("Missing modules for one or more dependencies marked as external modules: %s", missing_external_modules) # robot: look for existing dependencies, add them if robot and easyconfigs: # rely on EasyBuild module naming scheme when resolving dependencies, since we know that will # generate sensible module names that include the necessary information for the resolution to work # (name, version, toolchain, versionsuffix) being_installed = [EasyBuildMNS().det_full_module_name(p['ec']) for p in easyconfigs] additional = [] for entry in easyconfigs: # do not choose an entry that is being installed in the current run # if they depend, you probably want to rebuild them using the new dependency deps = entry['dependencies'] candidates = [d for d in deps if not EasyBuildMNS().det_full_module_name(d) in being_installed] if candidates: cand_dep = candidates[0] # find easyconfig, might not find any _log.debug("Looking for easyconfig for %s" % str(cand_dep)) # note: robot_find_easyconfig may return None path = robot_find_easyconfig(cand_dep['name'], det_full_ec_version(cand_dep)) if path is None: # no easyconfig found for dependency, add to list of irresolvable dependencies if cand_dep not in irresolvable: _log.debug("Irresolvable dependency found: %s" % cand_dep) irresolvable.append(cand_dep) # remove irresolvable dependency from list of dependencies so we can continue entry['dependencies'].remove(cand_dep) else: _log.info("Robot: resolving dependency %s with %s" % (cand_dep, path)) # build specs should not be passed down to resolved dependencies, # to avoid that e.g. --try-toolchain trickles down into the used toolchain itself hidden = cand_dep.get('hidden', False) processed_ecs = process_easyconfig(path, validate=not retain_all_deps, hidden=hidden) # ensure that selected easyconfig provides required dependency mods = [spec['ec'].full_mod_name for spec in processed_ecs] dep_mod_name = ActiveMNS().det_full_module_name(cand_dep) if not dep_mod_name in mods: raise EasyBuildError("easyconfig file %s does not contain module %s (mods: %s)", path, dep_mod_name, mods) for ec in processed_ecs: if not ec in easyconfigs + additional: additional.append(ec) _log.debug("Added %s as dependency of %s" % (ec, entry)) else: mod_name = EasyBuildMNS().det_full_module_name(entry['ec']) _log.debug("No more candidate dependencies to resolve for %s" % mod_name) # add additional (new) easyconfigs to list of stuff to process easyconfigs.extend(additional) _log.debug("Unprocessed dependencies: %s", easyconfigs) elif not robot: # no use in continuing if robot is not enabled, dependencies won't be resolved anyway irresolvable = [dep for x in easyconfigs for dep in x['dependencies']] break if irresolvable: _log.warning("Irresolvable dependencies (details): %s" % irresolvable) irresolvable_mods_eb = [EasyBuildMNS().det_full_module_name(dep) for dep in irresolvable] _log.warning("Irresolvable dependencies (EasyBuild module names): %s" % ', '.join(irresolvable_mods_eb)) irresolvable_mods = [ActiveMNS().det_full_module_name(dep) for dep in irresolvable] raise EasyBuildError("Irresolvable dependencies encountered: %s", ', '.join(irresolvable_mods)) _log.info("Dependency resolution complete, building as follows: %s" % ordered_ecs) return ordered_ecs