Пример #1
0
    def test_find_minimally_resolved_modules(self):
        """Test find_minimally_resolved_modules function."""

        # replace log.experimental with log.warning to allow experimental code
        easybuild.framework.easyconfig.tools._log.experimental = easybuild.framework.easyconfig.tools._log.warning

        test_easyconfigs = os.path.join(
            os.path.dirname(os.path.abspath(__file__)), 'easyconfigs')
        init_config(
            build_options={
                'valid_module_classes': module_classes(),
                'robot_path': test_easyconfigs,
            })

        barec = os.path.join(self.test_prefix, 'bar-1.2.3-goolf-1.4.10.eb')
        barec_txt = '\n'.join([
            "easyblock = 'ConfigureMake'",
            "name = 'bar'",
            "version = '1.2.3'",
            "homepage = 'http://example.com'",
            "description = 'foo'",
            "toolchain = {'name': 'goolf', 'version': '1.4.10'}",
            # deliberately listing components of toolchain as dependencies without specifying subtoolchains,
            # to test resolving of dependencies with minimal toolchain
            # for each of these, we know test easyconfigs are available (which are required here)
            "dependencies = [",
            "   ('OpenMPI', '1.6.4'),",  # available with GCC/4.7.2
            "   ('OpenBLAS', '0.2.6', '-LAPACK-3.4.2'),",  # available with gompi/1.4.10
            "   ('ScaLAPACK', '2.0.2', '-OpenBLAS-0.2.6-LAPACK-3.4.2'),",  # available with gompi/1.4.10
            "   ('SQLite', '3.8.10.2'),",  # only available with goolf/1.4.10
            "]",
        ])
        write_file(barec, barec_txt)
        bar = process_easyconfig(barec)[0]

        ecs = [bar]
        mods = [
            'gompi/1.4.10',
            'goolf/1.4.10',
            # include modules for dependencies, with subtoolchains rather than full toolchain (except for SQLite)
            'OpenMPI/1.6.4-GCC-4.7.2',
            'OpenBLAS/0.2.6-gompi-1.4.10-LAPACK-3.4.2',
            'ScaLAPACK/2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2',
            'SQLite/3.8.10.2-GCC-4.7.2',
        ]
        ordered_ecs, new_easyconfigs, new_avail_modules = find_minimally_resolved_modules(
            ecs, mods, [])

        # all dependencies are resolved for easyconfigs included in ordered_ecs
        self.assertEqual(len(ordered_ecs), 1)
        self.assertEqual(ordered_ecs[0]['dependencies'], [])

        # module is added to list of available modules
        self.assertTrue(bar['ec'].full_mod_name in new_avail_modules)

        # nothing left
        self.assertEqual(new_easyconfigs, [])
Пример #2
0
    def test_find_minimally_resolved_modules(self):
        """Test find_minimally_resolved_modules function."""

        # replace log.experimental with log.warning to allow experimental code
        easybuild.framework.easyconfig.tools._log.experimental = easybuild.framework.easyconfig.tools._log.warning

        test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs')
        init_config(build_options={
            'valid_module_classes': module_classes(),
            'robot_path': test_easyconfigs,
        })

        barec = os.path.join(self.test_prefix, 'bar-1.2.3-goolf-1.4.10.eb')
        barec_txt = '\n'.join([
            "easyblock = 'ConfigureMake'",
            "name = 'bar'",
            "version = '1.2.3'",
            "homepage = 'http://example.com'",
            "description = 'foo'",
            "toolchain = {'name': 'goolf', 'version': '1.4.10'}",
            # deliberately listing components of toolchain as dependencies without specifying subtoolchains,
            # to test resolving of dependencies with minimal toolchain
            # for each of these, we know test easyconfigs are available (which are required here)
            "dependencies = [",
            "   ('OpenMPI', '1.6.4'),",  # available with GCC/4.7.2
            "   ('OpenBLAS', '0.2.6', '-LAPACK-3.4.2'),",  # available with gompi/1.4.10
            "   ('ScaLAPACK', '2.0.2', '-OpenBLAS-0.2.6-LAPACK-3.4.2'),",  # available with gompi/1.4.10
            "   ('SQLite', '3.8.10.2'),",  # only available with goolf/1.4.10
            "]",
        ])
        write_file(barec, barec_txt)
        bar = process_easyconfig(barec)[0]

        ecs = [bar]
        mods = [
            'gompi/1.4.10',
            'goolf/1.4.10',
            # include modules for dependencies, with subtoolchains rather than full toolchain (except for SQLite)
            'OpenMPI/1.6.4-GCC-4.7.2',
            'OpenBLAS/0.2.6-gompi-1.4.10-LAPACK-3.4.2',
            'ScaLAPACK/2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2',
            'SQLite/3.8.10.2-GCC-4.7.2',
        ]
        ordered_ecs, new_easyconfigs, new_avail_modules = find_minimally_resolved_modules(ecs, mods, [])

        # all dependencies are resolved for easyconfigs included in ordered_ecs
        self.assertEqual(len(ordered_ecs), 1)
        self.assertEqual(ordered_ecs[0]['dependencies'], [])

        # module is added to list of available modules
        self.assertTrue(bar['ec'].full_mod_name in new_avail_modules)

        # nothing left
        self.assertEqual(new_easyconfigs, [])
Пример #3
0
def resolve_dependencies(easyconfigs,
                         retain_all_deps=False,
                         minimal_toolchains=False,
                         use_existing_modules=False):
    """
    Work through the list of easyconfigs to determine an optimal order
    @param easyconfigs: list of easyconfigs
    @param retain_all_deps: boolean indicating whether all dependencies must be retained, regardless of availability;
                            retain all deps when True, check matching build option when False
    @param minimal_toolchains: boolean for whether to try to resolve dependencies with minimum possible toolchain
    @param use_existing_modules: boolean for whether to prioritise the reuse of existing modules (works in
                                     combination with minimal_toolchains)
    """

    robot = build_option('robot_path')
    # retain all dependencies if specified by either the resp. build option or the dedicated named argument
    retain_all_deps = build_option('retain_all_deps') or retain_all_deps

    existing_modules = modules_tool().available()
    if retain_all_deps:
        # assume that no modules are available when forced, to retain all dependencies
        avail_modules = []
        _log.info("Forcing all dependencies to be retained.")
    else:
        # Get a list of all available modules (format: [(name, installversion), ...])
        avail_modules = existing_modules[:]

        if len(avail_modules) == 0:
            _log.warning(
                "No installed modules. Your MODULEPATH is probably incomplete: %s"
                % os.getenv('MODULEPATH'))

    ordered_ecs = []
    # all available modules can be used for resolving dependencies except those that will be installed
    being_installed = [p['full_mod_name'] for p in easyconfigs]
    avail_modules = [m for m in avail_modules if not m in being_installed]

    _log.debug('easyconfigs before resolving deps: %s' % easyconfigs)

    # resolve all dependencies, put a safeguard in place to avoid an infinite loop (shouldn't occur though)
    irresolvable = []
    loopcnt = 0
    maxloopcnt = 10000
    while easyconfigs:
        # make sure this stops, we really don't want to get stuck in an infinite loop
        loopcnt += 1
        if loopcnt > maxloopcnt:
            raise EasyBuildError(
                "Maximum loop cnt %s reached, so quitting (easyconfigs: %s, irresolvable: %s)",
                maxloopcnt, easyconfigs, irresolvable)

        # first try resolving dependencies without using external dependencies
        last_processed_count = -1
        while len(avail_modules) > last_processed_count:
            last_processed_count = len(avail_modules)
            if minimal_toolchains:
                res = find_minimally_resolved_modules(
                    easyconfigs,
                    avail_modules,
                    existing_modules,
                    retain_all_deps=retain_all_deps,
                    use_existing_modules=use_existing_modules)
            else:
                res = find_resolved_modules(easyconfigs,
                                            avail_modules,
                                            retain_all_deps=retain_all_deps)
            more_ecs, easyconfigs, avail_modules = res
            for ec in more_ecs:
                if not ec['full_mod_name'] in [
                        x['full_mod_name'] for x in ordered_ecs
                ]:
                    ordered_ecs.append(ec)

        # dependencies marked as external modules should be resolved via available modules at this point
        missing_external_modules = [
            d['full_mod_name'] for ec in easyconfigs
            for d in ec['dependencies'] if d.get('external_module', False)
        ]
        if missing_external_modules:
            raise EasyBuildError(
                "Missing modules for one or more dependencies marked as external modules: %s",
                missing_external_modules)

        # robot: look for existing dependencies, add them
        if robot and easyconfigs:

            # rely on EasyBuild module naming scheme when resolving dependencies, since we know that will
            # generate sensible module names that include the necessary information for the resolution to work
            # (name, version, toolchain, versionsuffix)
            being_installed = [
                EasyBuildMNS().det_full_module_name(p['ec'])
                for p in easyconfigs
            ]

            additional = []
            for entry in easyconfigs:
                # do not choose an entry that is being installed in the current run
                # if they depend, you probably want to rebuild them using the new dependency
                deps = entry['dependencies']
                candidates = [
                    d for d in deps if not EasyBuildMNS().det_full_module_name(
                        d) in being_installed
                ]
                if candidates:
                    cand_dep = candidates[0]
                    # find easyconfig, might not find any
                    _log.debug("Looking for easyconfig for %s" % str(cand_dep))
                    # note: robot_find_easyconfig may return None
                    path = robot_find_easyconfig(cand_dep['name'],
                                                 det_full_ec_version(cand_dep))

                    if path is None:
                        # no easyconfig found for dependency, add to list of irresolvable dependencies
                        if cand_dep not in irresolvable:
                            _log.debug("Irresolvable dependency found: %s" %
                                       cand_dep)
                            irresolvable.append(cand_dep)
                        # remove irresolvable dependency from list of dependencies so we can continue
                        entry['dependencies'].remove(cand_dep)
                    else:
                        _log.info("Robot: resolving dependency %s with %s" %
                                  (cand_dep, path))
                        # build specs should not be passed down to resolved dependencies,
                        # to avoid that e.g. --try-toolchain trickles down into the used toolchain itself
                        hidden = cand_dep.get('hidden', False)
                        processed_ecs = process_easyconfig(
                            path, validate=not retain_all_deps, hidden=hidden)

                        # ensure that selected easyconfig provides required dependency
                        mods = [
                            spec['ec'].full_mod_name for spec in processed_ecs
                        ]
                        dep_mod_name = ActiveMNS().det_full_module_name(
                            cand_dep)
                        if not dep_mod_name in mods:
                            raise EasyBuildError(
                                "easyconfig file %s does not contain module %s (mods: %s)",
                                path, dep_mod_name, mods)

                        for ec in processed_ecs:
                            if not ec in easyconfigs + additional:
                                additional.append(ec)
                                _log.debug("Added %s as dependency of %s" %
                                           (ec, entry))
                else:
                    mod_name = EasyBuildMNS().det_full_module_name(entry['ec'])
                    _log.debug(
                        "No more candidate dependencies to resolve for %s" %
                        mod_name)

            # add additional (new) easyconfigs to list of stuff to process
            easyconfigs.extend(additional)
            _log.debug("Unprocessed dependencies: %s", easyconfigs)

        elif not robot:
            # no use in continuing if robot is not enabled, dependencies won't be resolved anyway
            irresolvable = [
                dep for x in easyconfigs for dep in x['dependencies']
            ]
            break

    if irresolvable:
        _log.warning("Irresolvable dependencies (details): %s" % irresolvable)
        irresolvable_mods_eb = [
            EasyBuildMNS().det_full_module_name(dep) for dep in irresolvable
        ]
        _log.warning("Irresolvable dependencies (EasyBuild module names): %s" %
                     ', '.join(irresolvable_mods_eb))
        irresolvable_mods = [
            ActiveMNS().det_full_module_name(dep) for dep in irresolvable
        ]
        raise EasyBuildError("Irresolvable dependencies encountered: %s",
                             ', '.join(irresolvable_mods))

    _log.info("Dependency resolution complete, building as follows: %s" %
              ordered_ecs)
    return ordered_ecs
Пример #4
0
def resolve_dependencies(easyconfigs, retain_all_deps=False, minimal_toolchains=False, use_existing_modules=False):
    """
    Work through the list of easyconfigs to determine an optimal order
    @param easyconfigs: list of easyconfigs
    @param retain_all_deps: boolean indicating whether all dependencies must be retained, regardless of availability;
                            retain all deps when True, check matching build option when False
    @param minimal_toolchains: boolean for whether to try to resolve dependencies with minimum possible toolchain
    @param use_existing_modules: boolean for whether to prioritise the reuse of existing modules (works in
                                     combination with minimal_toolchains)
    """

    robot = build_option('robot_path')
    # retain all dependencies if specified by either the resp. build option or the dedicated named argument
    retain_all_deps = build_option('retain_all_deps') or retain_all_deps

    existing_modules = modules_tool().available()
    if retain_all_deps:
        # assume that no modules are available when forced, to retain all dependencies
        avail_modules = []
        _log.info("Forcing all dependencies to be retained.")
    else:
        # Get a list of all available modules (format: [(name, installversion), ...])
        avail_modules = existing_modules[:]

        if len(avail_modules) == 0:
            _log.warning("No installed modules. Your MODULEPATH is probably incomplete: %s" % os.getenv('MODULEPATH'))

    ordered_ecs = []
    # all available modules can be used for resolving dependencies except those that will be installed
    being_installed = [p['full_mod_name'] for p in easyconfigs]
    avail_modules = [m for m in avail_modules if not m in being_installed]

    _log.debug('easyconfigs before resolving deps: %s' % easyconfigs)

    # resolve all dependencies, put a safeguard in place to avoid an infinite loop (shouldn't occur though)
    irresolvable = []
    loopcnt = 0
    maxloopcnt = 10000
    while easyconfigs:
        # make sure this stops, we really don't want to get stuck in an infinite loop
        loopcnt += 1
        if loopcnt > maxloopcnt:
            raise EasyBuildError("Maximum loop cnt %s reached, so quitting (easyconfigs: %s, irresolvable: %s)",
                                 maxloopcnt, easyconfigs, irresolvable)

        # first try resolving dependencies without using external dependencies
        last_processed_count = -1
        while len(avail_modules) > last_processed_count:
            last_processed_count = len(avail_modules)
            if minimal_toolchains:
                res = find_minimally_resolved_modules(easyconfigs, avail_modules, existing_modules,
                                                      retain_all_deps=retain_all_deps,
                                                      use_existing_modules=use_existing_modules)
            else:
                res = find_resolved_modules(easyconfigs, avail_modules, retain_all_deps=retain_all_deps)
            more_ecs, easyconfigs, avail_modules = res
            for ec in more_ecs:
                if not ec['full_mod_name'] in [x['full_mod_name'] for x in ordered_ecs]:
                    ordered_ecs.append(ec)

        # dependencies marked as external modules should be resolved via available modules at this point
        missing_external_modules = [d['full_mod_name'] for ec in easyconfigs for d in ec['dependencies']
                                    if d.get('external_module', False)]
        if missing_external_modules:
            raise EasyBuildError("Missing modules for one or more dependencies marked as external modules: %s",
                                 missing_external_modules)

        # robot: look for existing dependencies, add them
        if robot and easyconfigs:

            # rely on EasyBuild module naming scheme when resolving dependencies, since we know that will
            # generate sensible module names that include the necessary information for the resolution to work
            # (name, version, toolchain, versionsuffix)
            being_installed = [EasyBuildMNS().det_full_module_name(p['ec']) for p in easyconfigs]

            additional = []
            for entry in easyconfigs:
                # do not choose an entry that is being installed in the current run
                # if they depend, you probably want to rebuild them using the new dependency
                deps = entry['dependencies']
                candidates = [d for d in deps if not EasyBuildMNS().det_full_module_name(d) in being_installed]
                if candidates:
                    cand_dep = candidates[0]
                    # find easyconfig, might not find any
                    _log.debug("Looking for easyconfig for %s" % str(cand_dep))
                    # note: robot_find_easyconfig may return None
                    path = robot_find_easyconfig(cand_dep['name'], det_full_ec_version(cand_dep))

                    if path is None:
                        # no easyconfig found for dependency, add to list of irresolvable dependencies
                        if cand_dep not in irresolvable:
                            _log.debug("Irresolvable dependency found: %s" % cand_dep)
                            irresolvable.append(cand_dep)
                        # remove irresolvable dependency from list of dependencies so we can continue
                        entry['dependencies'].remove(cand_dep)
                    else:
                        _log.info("Robot: resolving dependency %s with %s" % (cand_dep, path))
                        # build specs should not be passed down to resolved dependencies,
                        # to avoid that e.g. --try-toolchain trickles down into the used toolchain itself
                        hidden = cand_dep.get('hidden', False)
                        processed_ecs = process_easyconfig(path, validate=not retain_all_deps, hidden=hidden)

                        # ensure that selected easyconfig provides required dependency
                        mods = [spec['ec'].full_mod_name for spec in processed_ecs]
                        dep_mod_name = ActiveMNS().det_full_module_name(cand_dep)
                        if not dep_mod_name in mods:
                            raise EasyBuildError("easyconfig file %s does not contain module %s (mods: %s)",
                                                 path, dep_mod_name, mods)

                        for ec in processed_ecs:
                            if not ec in easyconfigs + additional:
                                additional.append(ec)
                                _log.debug("Added %s as dependency of %s" % (ec, entry))
                else:
                    mod_name = EasyBuildMNS().det_full_module_name(entry['ec'])
                    _log.debug("No more candidate dependencies to resolve for %s" % mod_name)

            # add additional (new) easyconfigs to list of stuff to process
            easyconfigs.extend(additional)
            _log.debug("Unprocessed dependencies: %s", easyconfigs)

        elif not robot:
            # no use in continuing if robot is not enabled, dependencies won't be resolved anyway
            irresolvable = [dep for x in easyconfigs for dep in x['dependencies']]
            break

    if irresolvable:
        _log.warning("Irresolvable dependencies (details): %s" % irresolvable)
        irresolvable_mods_eb = [EasyBuildMNS().det_full_module_name(dep) for dep in irresolvable]
        _log.warning("Irresolvable dependencies (EasyBuild module names): %s" % ', '.join(irresolvable_mods_eb))
        irresolvable_mods = [ActiveMNS().det_full_module_name(dep) for dep in irresolvable]
        raise EasyBuildError("Irresolvable dependencies encountered: %s", ', '.join(irresolvable_mods))

    _log.info("Dependency resolution complete, building as follows: %s" % ordered_ecs)
    return ordered_ecs