def add_module(self, modules):
        """
        Check if module exist, if so add to list.
        """
        self.log.deprecated("Use of add_module function should be replaced by load([<list of modules>])", '2.0')
        for mod in modules:
            if isinstance(mod, (list, tuple)):
                mod_dict = {
                    'name': mod[0],
                    'version': mod[1],
                    'versionsuffix': '',
                    'toolchain': {
                        'name': DUMMY_TOOLCHAIN_NAME,
                        'version': DUMMY_TOOLCHAIN_VERSION,
                    },
                }
                mod_name = det_full_module_name(mod_dict)
            elif isinstance(mod, basestring):
                mod_name = mod
            elif isinstance(mod, dict):
                mod_name = det_full_module_name(mod)
            else:
                self.log.error("Can't add module %s: unknown type" % str(mod))

            mods = self.available(mod_name)
            if mod_name in mods:
                # ok
                self._modules.append(mod_name)
            else:
                if len(mods) == 0:
                    self.log.warning('No module %s available' % str(mod))
                else:
                    self.log.warning('More than one module found for %s: %s' % (mod, mods))
                continue
    def test_buildininstalldir(self):
        """Test specifying build in install dir."""
        config.variables['buildpath'] = tempfile.mkdtemp()
        config.variables['installpath'] = tempfile.mkdtemp()
        self.contents = '\n'.join([
            'name = "pi"',
            'version = "3.14"',
            'homepage = "http://example.com"',
            'description = "test easyconfig"',
            'toolchain = {"name": "dummy", "version": "dummy"}',
            'buildininstalldir = True',
        ])
        self.prep()
        eb = EasyBlock(self.eb_file)
        eb.gen_builddir()
        eb.mod_name = det_full_module_name(eb.cfg)  # required by gen_installdir()
        eb.gen_installdir()
        eb.make_builddir()
        eb.make_installdir()
        self.assertEqual(eb.builddir, eb.installdir)
        self.assertTrue(os.path.isdir(eb.builddir))

        # cleanup
        shutil.rmtree(config.variables['buildpath'])
        shutil.rmtree(config.variables['installpath'])
def process_easyconfig(path, build_options=None, build_specs=None):
    """
    Process easyconfig, returning some information for each block
    @param path: path to easyconfig file
    @param build_options: dictionary specifying build options (e.g. robot_path, check_osdeps, ...)
    @param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...)
    """
    blocks = retrieve_blocks_in_spec(path, build_options.get("only_blocks", None))

    easyconfigs = []
    for spec in blocks:
        # process for dependencies and real installversionname
        _log.debug("Processing easyconfig %s" % spec)

        # create easyconfig
        try:
            ec = EasyConfig(spec, build_options=build_options, build_specs=build_specs)
        except EasyBuildError, err:
            msg = "Failed to process easyconfig %s:\n%s" % (spec, err.msg)
            _log.exception(msg)

        name = ec["name"]

        # this app will appear as following module in the list
        easyconfig = {
            "ec": ec,
            "spec": spec,
            "module": det_full_module_name(ec),
            "dependencies": [],
            "builddependencies": [],
        }
        if len(blocks) > 1:
            easyconfig["originalSpec"] = path

        # add build dependencies
        for dep in ec.builddependencies():
            _log.debug("Adding build dependency %s for app %s." % (dep, name))
            easyconfig["builddependencies"].append(dep)

        # add dependencies (including build dependencies)
        for dep in ec.dependencies():
            _log.debug("Adding dependency %s for app %s." % (dep, name))
            easyconfig["dependencies"].append(dep)

        # add toolchain as dependency too
        if ec.toolchain.name != DUMMY_TOOLCHAIN_NAME:
            dep = ec.toolchain.as_dict()
            _log.debug("Adding toolchain %s as dependency for app %s." % (dep, name))
            easyconfig["dependencies"].append(dep)

        del ec

        # this is used by the parallel builder
        easyconfig["unresolved_deps"] = copy.deepcopy(easyconfig["dependencies"])

        easyconfigs.append(easyconfig)
 def add_dependencies(self, dependencies):
     """ Verify if the given dependencies exist and add them """
     self.log.debug("add_dependencies: adding toolchain dependencies %s" % dependencies)
     for dep in dependencies:
         mod_name = det_full_module_name(dep)
         if not self.modules_tool.exists(mod_name):
             self.log.error("add_dependencies: no module found for dependency %s" % str(dep))
         else:
             self.dependencies.append(dep)
             self.log.debug("add_dependencies: added toolchain dependency %s" % dep)
Exemple #5
0
def process_easyconfig(path, onlyBlocks=None, regtest_online=False, validate=True):
    """
    Process easyconfig, returning some information for each block
    """
    blocks = retrieve_blocks_in_spec(path, onlyBlocks)

    easyconfigs = []
    for spec in blocks:
        # process for dependencies and real installversionname
        # - use mod? __init__ and importCfg are ignored.
        _log.debug("Processing easyconfig %s" % spec)

        # create easyconfig
        try:
            all_stops = [x[0] for x in EasyBlock.get_steps()]
            ec = EasyConfig(spec, validate=validate, valid_module_classes=module_classes(), valid_stops=all_stops)
        except EasyBuildError, err:
            msg = "Failed to process easyconfig %s:\n%s" % (spec, err.msg)
            _log.exception(msg)

        name = ec['name']

        # this app will appear as following module in the list
        easyconfig = {
            'ec': ec,
            'spec': spec,
            'module': det_full_module_name(ec),
            'dependencies': [],
            'builddependencies': [],
        }
        if len(blocks) > 1:
            easyconfig['originalSpec'] = path

        # add build dependencies
        for dep in ec.builddependencies():
            _log.debug("Adding build dependency %s for app %s." % (dep, name))
            easyconfig['builddependencies'].append(dep)

        # add dependencies (including build dependencies)
        for dep in ec.dependencies():
            _log.debug("Adding dependency %s for app %s." % (dep, name))
            easyconfig['dependencies'].append(dep)

        # add toolchain as dependency too
        if ec.toolchain.name != DUMMY_TOOLCHAIN_NAME:
            dep = ec.toolchain.as_dict()
            _log.debug("Adding toolchain %s as dependency for app %s." % (dep, name))
            easyconfig['dependencies'].append(dep)

        del ec

        # this is used by the parallel builder
        easyconfig['unresolved_deps'] = copy.deepcopy(easyconfig['dependencies'])

        easyconfigs.append(easyconfig)
Exemple #6
0
def create_job(build_command, easyconfig, output_dir="", conn=None, ppn=None):
    """
    Creates a job, to build a *single* easyconfig
    build_command is a format string in which a full path to an eb file will be substituted
    easyconfig should be in the format as processEasyConfig returns them
    output_dir is an optional path. EASYBUILDTESTOUTPUT will be set inside the job with this variable
    returns the job
    """
    # create command based on build_command template
    command = build_command % {'spec': easyconfig['spec']}

    # capture PYTHONPATH, MODULEPATH and all variables starting with EASYBUILD
    easybuild_vars = {}
    for name in os.environ:
        if name.startswith("EASYBUILD"):
            easybuild_vars[name] = os.environ[name]

    others = ["PYTHONPATH", "MODULEPATH"]

    for env_var in others:
        if env_var in os.environ:
            easybuild_vars[env_var] = os.environ[env_var]

    _log.info("Dictionary of environment variables passed to job: %s" %
              easybuild_vars)

    # obtain unique name based on name/easyconfig version tuple
    ec_tuple = (easyconfig['ec']['name'],
                det_full_ec_version(easyconfig['ec']))
    name = '-'.join(ec_tuple)

    var = config.oldstyle_environment_variables['test_output_path']
    easybuild_vars[var] = os.path.join(os.path.abspath(output_dir), name)

    # just use latest build stats
    repo = init_repository(get_repository(), get_repositorypath())
    buildstats = repo.get_buildstats(*ec_tuple)
    resources = {}
    if buildstats:
        previous_time = buildstats[-1]['build_time']
        resources['hours'] = int(math.ceil(previous_time * 2 / 60))

    job = PbsJob(command,
                 name,
                 easybuild_vars,
                 resources=resources,
                 conn=conn,
                 ppn=ppn)
    job.module = det_full_module_name(easyconfig['ec'])

    return job
 def add_dependencies(self, dependencies):
     """ Verify if the given dependencies exist and add them """
     self.log.debug("add_dependencies: adding toolchain dependencies %s" %
                    dependencies)
     for dep in dependencies:
         mod_name = det_full_module_name(dep)
         if not self.modules_tool.exists(mod_name):
             self.log.error(
                 'add_dependencies: no module found for dependency %s' %
                 str(dep))
         else:
             self.dependencies.append(dep)
             self.log.debug(
                 'add_dependencies: added toolchain dependency %s' % dep)
Exemple #8
0
    def add_module(self, modules):
        """
        Check if module exist, if so add to list.
        """
        self.log.deprecated(
            "Use of add_module function should be replaced by load([<list of modules>])",
            '2.0')
        for mod in modules:
            if isinstance(mod, (list, tuple)):
                mod_dict = {
                    'name': mod[0],
                    'version': mod[1],
                    'versionsuffix': '',
                    'toolchain': {
                        'name': DUMMY_TOOLCHAIN_NAME,
                        'version': DUMMY_TOOLCHAIN_VERSION,
                    },
                }
                mod_name = det_full_module_name(mod_dict)
            elif isinstance(mod, basestring):
                mod_name = mod
            elif isinstance(mod, dict):
                mod_name = det_full_module_name(mod)
            else:
                self.log.error("Can't add module %s: unknown type" % str(mod))

            mods = self.available(mod_name)
            if mod_name in mods:
                # ok
                self._modules.append(mod_name)
            else:
                if len(mods) == 0:
                    self.log.warning('No module %s available' % str(mod))
                else:
                    self.log.warning('More then one module found for %s: %s' %
                                     (mod, mods))
                continue
def create_job(build_command, easyconfig, output_dir=None, conn=None, ppn=None):
    """
    Creates a job, to build a *single* easyconfig
    @param build_command: format string for command, full path to an easyconfig file will be substituted in it
    @param easyconfig: easyconfig as processed by process_easyconfig
    @param output_dir: optional output path; $EASYBUILDTESTOUTPUT will be set inside the job with this variable
    @param conn: open connection to PBS server
    @param ppn: ppn setting to use (# 'processors' (cores) per node to use)
    returns the job
    """
    if output_dir is None:
        output_dir = 'easybuild-build'

    # create command based on build_command template
    command = build_command % {'spec': easyconfig['spec']}

    # capture PYTHONPATH, MODULEPATH and all variables starting with EASYBUILD
    easybuild_vars = {}
    for name in os.environ:
        if name.startswith("EASYBUILD"):
            easybuild_vars[name] = os.environ[name]

    others = ["PYTHONPATH", "MODULEPATH"]

    for env_var in others:
        if env_var in os.environ:
            easybuild_vars[env_var] = os.environ[env_var]

    _log.info("Dictionary of environment variables passed to job: %s" % easybuild_vars)

    # obtain unique name based on name/easyconfig version tuple
    ec_tuple = (easyconfig['ec']['name'], det_full_ec_version(easyconfig['ec']))
    name = '-'.join(ec_tuple)

    var = config.oldstyle_environment_variables['test_output_path']
    easybuild_vars[var] = os.path.join(os.path.abspath(output_dir), name)

    # just use latest build stats
    repo = init_repository(get_repository(), get_repositorypath())
    buildstats = repo.get_buildstats(*ec_tuple)
    resources = {}
    if buildstats:
        previous_time = buildstats[-1]['build_time']
        resources['hours'] = int(math.ceil(previous_time * 2 / 60))

    job = PbsJob(command, name, easybuild_vars, resources=resources, conn=conn, ppn=ppn)
    job.module = det_full_module_name(easyconfig['ec'])

    return job
Exemple #10
0
    def test_gen_dirs(self):
        """Test methods that generate/set build/install directory names."""
        self.contents = '\n'.join([
            "name = 'pi'",
            "version = '3.14'",
            "homepage = 'http://example.com'",
            "description = 'test easyconfig'",
            "toolchain = {'name': 'dummy', 'version': 'dummy'}",
        ])
        self.writeEC()
        stdoutorig = sys.stdout
        sys.stdout = open("/dev/null", 'w')
        eb = EasyBlock(EasyConfig(self.eb_file))
        resb = eb.gen_builddir()
        eb.mod_name = det_full_module_name(
            eb.cfg)  # required by gen_installdir()
        resi = eb.gen_installdir()
        eb.make_builddir()
        eb.make_installdir()
        # doesn't return anything
        self.assertEqual(resb, None)
        self.assertEqual(resi, None)
        # directories are set, and exist
        self.assertTrue(os.path.isdir(eb.builddir))
        self.assertTrue(os.path.isdir(eb.installdir))

        # make sure cleaning up old build dir is default
        self.assertTrue(eb.cfg['cleanupoldbuild']
                        or eb.cfg.get('cleanupoldbuild', True))
        builddir = eb.builddir
        eb.gen_builddir()
        self.assertEqual(builddir, eb.builddir)
        eb.cfg['cleanupoldbuild'] = True
        eb.gen_builddir()
        self.assertEqual(builddir, eb.builddir)

        # make sure build dir is unique
        eb.cfg['cleanupoldbuild'] = False
        builddir = eb.builddir
        for i in range(3):
            eb.gen_builddir()
            self.assertEqual(eb.builddir, "%s.%d" % (builddir, i))
            eb.make_builddir()

        # cleanup
        sys.stdout.close()
        sys.stdout = stdoutorig
        eb.close_log()
Exemple #11
0
def print_dry_run(easyconfigs, robot=None):
    if robot is None:
        print_msg("Dry run: printing build status of easyconfigs")
        all_specs = easyconfigs
    else: 
        print_msg("Dry run: printing build status of easyconfigs and dependencies")
        all_specs = resolve_dependencies(easyconfigs, robot, True)
    unbuilt_specs = skip_available(all_specs, True)
    dry_run_fmt = "%3s %s (module: %s)"
    for spec in all_specs:
        if spec in unbuilt_specs:
            ans = '[ ]'
        else:
            ans = '[x]'
        mod = det_full_module_name(spec['ec'])
        print dry_run_fmt % (ans, spec['spec'], mod)
Exemple #12
0
def find_resolved_modules(unprocessed, processed):
    """
    Find easyconfigs in unprocessed which can be fully resolved using easyconfigs in processed
    """
    ordered_ecs = []

    for ec in unprocessed:
        ec['dependencies'] = [d for d in ec['dependencies'] if not det_full_module_name(d) in processed]

        if len(ec['dependencies']) == 0:
            _log.debug("Adding easyconfig %s to final list" % ec['spec'])
            ordered_ecs.append(ec)
            processed.append(ec['module'])

    unprocessed[:] = [m for m in unprocessed if len(m['dependencies']) > 0]

    return ordered_ecs
    def test_gen_dirs(self):
        """Test methods that generate/set build/install directory names."""
        self.contents = '\n'.join([
            "name = 'pi'",
            "version = '3.14'",
            "homepage = 'http://example.com'",
            "description = 'test easyconfig'",
            "toolchain = {'name': 'dummy', 'version': 'dummy'}",
        ])
        self.writeEC()
        stdoutorig = sys.stdout
        sys.stdout = open("/dev/null", 'w')
        eb = EasyBlock(EasyConfig(self.eb_file))
        resb = eb.gen_builddir()
        eb.mod_name = det_full_module_name(eb.cfg)  # required by gen_installdir()
        resi = eb.gen_installdir()
        eb.make_builddir()
        eb.make_installdir()
        # doesn't return anything
        self.assertEqual(resb, None)
        self.assertEqual(resi, None)
        # directories are set, and exist
        self.assertTrue(os.path.isdir(eb.builddir))
        self.assertTrue(os.path.isdir(eb.installdir))

        # make sure cleaning up old build dir is default
        self.assertTrue(eb.cfg['cleanupoldbuild'] or eb.cfg.get('cleanupoldbuild', True))
        builddir = eb.builddir
        eb.gen_builddir()
        self.assertEqual(builddir, eb.builddir)
        eb.cfg['cleanupoldbuild'] = True
        eb.gen_builddir()
        self.assertEqual(builddir, eb.builddir)

        # make sure build dir is unique
        eb.cfg['cleanupoldbuild'] = False
        builddir = eb.builddir
        for i in range(3):
            eb.gen_builddir()
            self.assertEqual(eb.builddir, "%s.%d" % (builddir, i))
            eb.make_builddir()

        # cleanup
        sys.stdout.close()
        sys.stdout = stdoutorig
        eb.close_log()
def create_job(build_command, easyconfig, output_dir="", conn=None, ppn=None):
    """
    Creates a job, to build a *single* easyconfig
    build_command is a format string in which a full path to an eb file will be substituted
    easyconfig should be in the format as processEasyConfig returns them
    output_dir is an optional path. EASYBUILDTESTOUTPUT will be set inside the job with this variable
    returns the job
    """
    # create command based on build_command template
    command = build_command % {'spec': easyconfig['spec']}

    # capture PYTHONPATH, MODULEPATH and all variables starting with EASYBUILD
    easybuild_vars = {}
    for name in os.environ:
        if name.startswith("EASYBUILD"):
            easybuild_vars[name] = os.environ[name]

    others = ["PYTHONPATH", "MODULEPATH"]

    for env_var in others:
        if env_var in os.environ:
            easybuild_vars[env_var] = os.environ[env_var]

    _log.info("Dictionary of environment variables passed to job: %s" % easybuild_vars)

    # obtain unique name based on name/easyconfig version tuple
    ec_tuple = (easyconfig['ec']['name'], det_full_ec_version(easyconfig['ec']))
    name = '-'.join(ec_tuple)

    var = config.oldstyle_environment_variables['test_output_path']
    easybuild_vars[var] = os.path.join(os.path.abspath(output_dir), name)

    # just use latest build stats
    repo = init_repository(get_repository(), get_repositorypath())
    buildstats = repo.get_buildstats(*ec_tuple)
    resources = {}
    if buildstats:
        previous_time = buildstats[-1]['build_time']
        resources['hours'] = int(math.ceil(previous_time * 2 / 60))

    job = PbsJob(command, name, easybuild_vars, resources=resources, conn=conn, ppn=ppn)
    job.module = det_full_module_name(easyconfig['ec'])

    return job
def print_dry_run(easyconfigs, short=False, build_options=None, build_specs=None):
    """
    Print dry run information
    @param easyconfigs: list of easyconfig files
    @param short: print short output (use a variable for the common prefix)
    @param build_options: dictionary specifying build options (e.g. robot_path, check_osdeps, ...)
    @param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...)
    """
    lines = []
    if build_options.get("robot_path", None) is None:
        lines.append("Dry run: printing build status of easyconfigs")
        all_specs = easyconfigs
    else:
        lines.append("Dry run: printing build status of easyconfigs and dependencies")
        build_options = copy.deepcopy(build_options)
        build_options.update({"retain_all_deps": True, "check_osdeps": False})
        all_specs = resolve_dependencies(easyconfigs, build_options=build_options, build_specs=build_specs)

    unbuilt_specs = skip_available(all_specs, testing=True)
    dry_run_fmt = " * [%1s] %s (module: %s)"  # markdown compatible (list of items with checkboxes in front)

    var_name = "CFGS"
    common_prefix = det_common_path_prefix([spec["spec"] for spec in all_specs])
    # only allow short if common prefix is long enough
    short = short and common_prefix is not None and len(common_prefix) > len(var_name) * 2
    for spec in all_specs:
        if spec in unbuilt_specs:
            ans = " "
        else:
            ans = "x"
        mod = det_full_module_name(spec["ec"])

        if short:
            item = os.path.join("$%s" % var_name, spec["spec"][len(common_prefix) + 1 :])
        else:
            item = spec["spec"]
        lines.append(dry_run_fmt % (ans, item, mod))

    if short:
        # insert after 'Dry run:' message
        lines.insert(1, "%s=%s" % (var_name, common_prefix))
    silent = build_options.get("silent", False)
    print_msg("\n".join(lines), log=_log, silent=silent, prefix=False)
Exemple #16
0
 def test_buildininstalldir(self):
     """Test specifying build in install dir."""
     self.contents = '\n'.join([
         'name = "pi"',
         'version = "3.14"',
         'homepage = "http://example.com"',
         'description = "test easyconfig"',
         'toolchain = {"name": "dummy", "version": "dummy"}',
         'buildininstalldir = True',
     ])
     self.prep()
     ec = EasyConfig(self.eb_file)
     eb = EasyBlock(ec)
     eb.gen_builddir()
     eb.mod_name = det_full_module_name(eb.cfg)  # required by gen_installdir()
     eb.gen_installdir()
     eb.make_builddir()
     eb.make_installdir()
     self.assertEqual(eb.builddir, eb.installdir)
     self.assertTrue(os.path.isdir(eb.builddir))
def find_resolved_modules(unprocessed, avail_modules):
    """
    Find easyconfigs in 1st argument which can be fully resolved using modules specified in 2nd argument
    """
    ordered_ecs = []
    new_avail_modules = avail_modules[:]
    new_unprocessed = []

    for ec in unprocessed:
        new_ec = ec.copy()
        new_ec["dependencies"] = [d for d in new_ec["dependencies"] if not det_full_module_name(d) in new_avail_modules]

        if len(new_ec["dependencies"]) == 0:
            _log.debug("Adding easyconfig %s to final list" % new_ec["spec"])
            ordered_ecs.append(new_ec)
            new_avail_modules.append(ec["module"])

        else:
            new_unprocessed.append(new_ec)

    return ordered_ecs, new_unprocessed, new_avail_modules
Exemple #18
0
def resolve_dependencies(unprocessed, robot, force=False):
    """
    Work through the list of easyconfigs to determine an optimal order
    enabling force results in retaining all dependencies and skipping validation of easyconfigs
    """

    if force:
        # assume that no modules are available when forced
        available_modules = []
        _log.info("Forcing all dependencies to be retained.")
    else:
        # Get a list of all available modules (format: [(name, installversion), ...])
        available_modules = modules_tool().available()

        if len(available_modules) == 0:
            _log.warning("No installed modules. Your MODULEPATH is probably incomplete: %s" % os.getenv('MODULEPATH'))

    ordered_ecs = []
    # All available modules can be used for resolving dependencies except
    # those that will be installed
    being_installed = [p['module'] for p in unprocessed]
    processed = [m for m in available_modules if not m in being_installed]

    _log.debug('unprocessed before resolving deps: %s' % unprocessed)

    # as long as there is progress in processing the modules, keep on trying
    loopcnt = 0
    maxloopcnt = 10000
    robot_add_dep = True
    while robot_add_dep:

        robot_add_dep = False

        # make sure this stops, we really don't want to get stuck in an infinite loop
        loopcnt += 1
        if loopcnt > maxloopcnt:
            msg = "Maximum loop cnt %s reached, so quitting." % maxloopcnt
            _log.error(msg)

        # first try resolving dependencies without using external dependencies
        last_processed_count = -1
        while len(processed) > last_processed_count:
            last_processed_count = len(processed)
            ordered_ecs.extend(find_resolved_modules(unprocessed, processed))

        # robot: look for an existing dependency, add one
        if robot and len(unprocessed) > 0:

            being_installed = [det_full_module_name(p['ec'], eb_ns=True) for p in unprocessed]

            for entry in unprocessed:
                # do not choose an entry that is being installed in the current run
                # if they depend, you probably want to rebuild them using the new dependency
                deps = entry['dependencies']
                candidates = [d for d in deps if not det_full_module_name(d, eb_ns=True) in being_installed]
                if len(candidates) > 0:
                    cand_dep = candidates[0]
                    # find easyconfig, might not find any
                    _log.debug("Looking for easyconfig for %s" % str(cand_dep))
                    path = robot_find_easyconfig(robot, cand_dep['name'], det_full_ec_version(cand_dep))

                else:
                    path = None
                    mod_name = det_full_module_name(entry['ec'], eb_ns=True)
                    _log.debug("No more candidate dependencies to resolve for %s" % mod_name)

                if path is not None:
                    cand_dep = candidates[0]
                    _log.info("Robot: resolving dependency %s with %s" % (cand_dep, path))

                    processed_ecs = process_easyconfig(path, validate=(not force))

                    # ensure that selected easyconfig provides required dependency
                    mods = [det_full_module_name(spec['ec']) for spec in processed_ecs]
                    dep_mod_name = det_full_module_name(cand_dep)
                    if not dep_mod_name in mods:
                        _log.error("easyconfig file %s does not contain module %s (mods: %s)" % (path, dep_mod_name, mods))

                    unprocessed.extend(processed_ecs)
                    robot_add_dep = True
                    break

    _log.debug('unprocessed after resolving deps: %s' % unprocessed)

    # there are dependencies that cannot be resolved
    if len(unprocessed) > 0:
        _log.debug("List of unresolved dependencies: %s" % unprocessed)
        missing_dependencies = []
        for ec in unprocessed:
            for dep in ec['dependencies']:
                missing_dependencies.append('%s for %s' % (det_full_module_name(dep, eb_ns=True), dep))

        msg = "Dependencies not met. Cannot resolve %s" % missing_dependencies
        _log.error(msg)

    _log.info("Dependency resolution complete, building as follows:\n%s" % ordered_ecs)
    return ordered_ecs
 def det_module_name(self, name=None, version=None):
     """Determine module name for this toolchain."""
     return det_full_module_name(self.as_dict(name, version))
 def mk_dep_mod_name(spec):
     return tuple(det_full_module_name(spec).split(os.path.sep))
 def tokey(dep):
     """Determine key for specified dependency."""
     return det_full_module_name(dep)
 def tokey(dep):
     """Determine key for specified dependency."""
     return det_full_module_name(dep)
    def prepare(self, onlymod=None):
        """
        Prepare a set of environment parameters based on name/version of toolchain
        - load modules for toolchain and dependencies
        - generate extra variables and set them in the environment

        onlymod: Boolean/string to indicate if the toolchain should only load the environment
        with module (True) or also set all other variables (False) like compiler CC etc
        (If string: comma separated list of variables that will be ignored).
        """
        if self.modules_tool is None:
            self.log.raiseException("No modules tool defined.")

        if not self._toolchain_exists():
            self.log.raiseException(
                "No module found for toolchain name '%s' (%s)" %
                (self.name, self.version))

        if self.name == DUMMY_TOOLCHAIN_NAME:
            if self.version == DUMMY_TOOLCHAIN_VERSION:
                self.log.info(
                    'prepare: toolchain dummy mode, dummy version; not loading dependencies'
                )
            else:
                self.log.info(
                    'prepare: toolchain dummy mode and loading dependencies')
                self.modules_tool.load(
                    [det_full_module_name(dep) for dep in self.dependencies])
            return

        # Load the toolchain and dependencies modules
        self.log.debug("Loading toolchain module and dependencies...")
        self.modules_tool.load([self.det_module_name()])
        self.modules_tool.load(
            [det_full_module_name(dep) for dep in self.dependencies])

        # determine direct toolchain dependencies
        mod_name = self.det_module_name()
        self.toolchain_dependencies = self.modules_tool.dependencies_for(
            mod_name, depth=0)
        self.log.debug('prepare: list of direct toolchain dependencies: %s' %
                       self.toolchain_dependencies)

        # verify whether elements in toolchain definition match toolchain deps specified by loaded toolchain module
        toolchain_module_deps = set([
            self.modules_tool.module_software_name(d)
            for d in self.toolchain_dependencies
        ])
        toolchain_elements_mod_names = set([
            y for x in dir(self) if x.endswith('_MODULE_NAME')
            for y in eval("self.%s" % x)
        ])
        # filter out toolchain name (e.g. 'GCC') from list of toolchain elements
        toolchain_elements_mod_names = set(
            [x for x in toolchain_elements_mod_names if not x == self.name])

        # filter out optional toolchain elements if they're not used in the module
        for mod_name in toolchain_elements_mod_names.copy():
            if not self.is_required(mod_name):
                if not mod_name in toolchain_module_deps:
                    self.log.debug(
                        "Removing optional module %s from list of toolchain elements."
                        % mod_name)
                    toolchain_elements_mod_names.remove(mod_name)

        self.log.debug(
            "List of toolchain dependency modules from loaded toolchain module: %s"
            % toolchain_module_deps)
        self.log.debug(
            "List of toolchain elements from toolchain definition: %s" %
            toolchain_elements_mod_names)

        if toolchain_module_deps == toolchain_elements_mod_names:
            self.log.info(
                "List of toolchain dependency modules and toolchain definition match!"
            )
        else:
            self.log.error("List of toolchain dependency modules and toolchain definition do not match " \
                           "(%s vs %s)" % (toolchain_module_deps, toolchain_elements_mod_names))

        # Generate the variables to be set
        self.set_variables()

        # set the variables
        # onlymod can be comma-separated string of variables not to be set
        if onlymod == True:
            self.log.debug(
                "prepare: do not set additional variables onlymod=%s" %
                onlymod)
            self.generate_vars()
        else:
            self.log.debug("prepare: set additional variables onlymod=%s" %
                           onlymod)

            # add LDFLAGS and CPPFLAGS from dependencies to self.vars
            self._add_dependency_variables()
            self.generate_vars()
            self._setenv_variables(onlymod)
def resolve_dependencies(unprocessed, build_options=None, build_specs=None):
    """
    Work through the list of easyconfigs to determine an optimal order
    @param unprocessed: list of easyconfigs
    @param build_options: dictionary specifying build options (e.g. robot_path, check_osdeps, ...)
    @param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...)
    """

    robot = build_options.get("robot_path", None)

    if build_options.get("retain_all_deps", False):
        # assume that no modules are available when forced, to retain all dependencies
        avail_modules = []
        _log.info("Forcing all dependencies to be retained.")
    else:
        # Get a list of all available modules (format: [(name, installversion), ...])
        avail_modules = modules_tool().available()

        if len(avail_modules) == 0:
            _log.warning("No installed modules. Your MODULEPATH is probably incomplete: %s" % os.getenv("MODULEPATH"))

    ordered_ecs = []
    # all available modules can be used for resolving dependencies except those that will be installed
    being_installed = [p["module"] for p in unprocessed]
    avail_modules = [m for m in avail_modules if not m in being_installed]

    _log.debug("unprocessed before resolving deps: %s" % unprocessed)

    # resolve all dependencies, put a safeguard in place to avoid an infinite loop (shouldn't occur though)
    irresolvable = []
    loopcnt = 0
    maxloopcnt = 10000
    while unprocessed:
        # make sure this stops, we really don't want to get stuck in an infinite loop
        loopcnt += 1
        if loopcnt > maxloopcnt:
            tup = (maxloopcnt, unprocessed, irresolvable)
            msg = "Maximum loop cnt %s reached, so quitting (unprocessed: %s, irresolvable: %s)" % tup
            _log.error(msg)

        # first try resolving dependencies without using external dependencies
        last_processed_count = -1
        while len(avail_modules) > last_processed_count:
            last_processed_count = len(avail_modules)
            more_ecs, unprocessed, avail_modules = find_resolved_modules(unprocessed, avail_modules)
            for ec in more_ecs:
                if not ec["module"] in [x["module"] for x in ordered_ecs]:
                    ordered_ecs.append(ec)

        # robot: look for existing dependencies, add them
        if robot and unprocessed:

            being_installed = [det_full_module_name(p["ec"], eb_ns=True) for p in unprocessed]

            additional = []
            for i, entry in enumerate(unprocessed):
                # do not choose an entry that is being installed in the current run
                # if they depend, you probably want to rebuild them using the new dependency
                deps = entry["dependencies"]
                candidates = [d for d in deps if not det_full_module_name(d, eb_ns=True) in being_installed]
                if len(candidates) > 0:
                    cand_dep = candidates[0]
                    # find easyconfig, might not find any
                    _log.debug("Looking for easyconfig for %s" % str(cand_dep))
                    # note: robot_find_easyconfig may return None
                    path = robot_find_easyconfig(robot, cand_dep["name"], det_full_ec_version(cand_dep))

                    if path is None:
                        # no easyconfig found for dependency, add to list of irresolvable dependencies
                        if cand_dep not in irresolvable:
                            irresolvable.append(cand_dep)
                        # remove irresolvable dependency from list of dependencies so we can continue
                        entry["dependencies"].remove(cand_dep)
                    else:
                        _log.info("Robot: resolving dependency %s with %s" % (cand_dep, path))
                        processed_ecs = process_easyconfig(path, build_options=build_options, build_specs=build_specs)

                        # ensure that selected easyconfig provides required dependency
                        mods = [det_full_module_name(spec["ec"]) for spec in processed_ecs]
                        dep_mod_name = det_full_module_name(cand_dep)
                        if not dep_mod_name in mods:
                            tup = (path, dep_mod_name, mods)
                            _log.error("easyconfig file %s does not contain module %s (mods: %s)" % tup)

                        for ec in processed_ecs:
                            if not ec in unprocessed + additional:
                                additional.append(ec)
                                _log.debug("Added %s as dependency of %s" % (ec, entry))
                else:
                    mod_name = det_full_module_name(entry["ec"], eb_ns=True)
                    _log.debug("No more candidate dependencies to resolve for %s" % mod_name)

            # add additional (new) easyconfigs to list of stuff to process
            unprocessed.extend(additional)

    if irresolvable:
        irresolvable_mod_deps = [(det_full_module_name(dep, eb_ns=True), dep) for dep in irresolvable]
        _log.error("Irresolvable dependencies encountered: %s" % irresolvable_mod_deps)

    _log.info("Dependency resolution complete, building as follows:\n%s" % ordered_ecs)
    return ordered_ecs
Exemple #25
0
        _log.debug("Build stats: %s" % buildstats)

        if app.cfg['stop']:
            ended = "STOPPED"
            newLogDir = os.path.join(app.builddir, config.log_path())
        else:
            newLogDir = os.path.join(app.installdir, config.log_path())

            try:
                # upload spec to central repository
                repo = init_repository(get_repository(), get_repositorypath())
                if 'originalSpec' in module:
                    block = det_full_ec_version(app.cfg) + ".block"
                    repo.add_easyconfig(module['originalSpec'], app.name, block, buildstats, currentbuildstats)
                repo.add_easyconfig(spec, app.name, det_full_ec_version(app.cfg), buildstats, currentbuildstats)
                repo.commit("Built %s" % det_full_module_name(app.cfg))
                del repo
            except EasyBuildError, err:
                _log.warn("Unable to commit easyconfig to repository: %s", err)

        exitCode = 0
        succ = "successfully"
        summary = "COMPLETED"

        # cleanup logs
        app.close_log()
        try:
            if not os.path.isdir(newLogDir):
                os.makedirs(newLogDir)
            applicationLog = os.path.join(newLogDir, get_log_filename(app.name, app.version))
            shutil.move(app.logfile, applicationLog)
 def det_module_name(self, name=None, version=None):
     """Determine module name for this toolchain."""
     return det_full_module_name(self.as_dict(name, version))
 def mk_node_name(spec):
     if omit_versions:
         return spec["name"]
     else:
         return det_full_module_name(spec)
Exemple #28
0
 def mk_dep_mod_name(spec):
     return tuple(det_full_module_name(spec).split(os.path.sep))
    def prepare(self, onlymod=None):
        """
        Prepare a set of environment parameters based on name/version of toolchain
        - load modules for toolchain and dependencies
        - generate extra variables and set them in the environment

        onlymod: Boolean/string to indicate if the toolchain should only load the environment
        with module (True) or also set all other variables (False) like compiler CC etc
        (If string: comma separated list of variables that will be ignored).
        """
        if self.modules_tool is None:
            self.log.raiseException("No modules tool defined.")

        if not self._toolchain_exists():
            self.log.raiseException("No module found for toolchain name '%s' (%s)" % (self.name, self.version))

        if self.name == DUMMY_TOOLCHAIN_NAME:
            if self.version == DUMMY_TOOLCHAIN_VERSION:
                self.log.info("prepare: toolchain dummy mode, dummy version; not loading dependencies")
            else:
                self.log.info("prepare: toolchain dummy mode and loading dependencies")
                self.modules_tool.load([det_full_module_name(dep) for dep in self.dependencies])
            return

        # Load the toolchain and dependencies modules
        self.log.debug("Loading toolchain module and dependencies...")
        self.modules_tool.load([self.det_module_name()])
        self.modules_tool.load([det_full_module_name(dep) for dep in self.dependencies])

        # determine direct toolchain dependencies
        mod_name = self.det_module_name()
        self.toolchain_dependencies = self.modules_tool.dependencies_for(mod_name, depth=0)
        self.log.debug("prepare: list of direct toolchain dependencies: %s" % self.toolchain_dependencies)

        # verify whether elements in toolchain definition match toolchain deps specified by loaded toolchain module
        toolchain_module_deps = set([self.modules_tool.module_software_name(d) for d in self.toolchain_dependencies])
        toolchain_elements_mod_names = set(
            [y for x in dir(self) if x.endswith("_MODULE_NAME") for y in eval("self.%s" % x)]
        )
        # filter out toolchain name (e.g. 'GCC') from list of toolchain elements
        toolchain_elements_mod_names = set([x for x in toolchain_elements_mod_names if not x == self.name])

        # filter out optional toolchain elements if they're not used in the module
        for mod_name in toolchain_elements_mod_names.copy():
            if not self.is_required(mod_name):
                if not mod_name in toolchain_module_deps:
                    self.log.debug("Removing optional module %s from list of toolchain elements." % mod_name)
                    toolchain_elements_mod_names.remove(mod_name)

        self.log.debug("List of toolchain dependency modules from loaded toolchain module: %s" % toolchain_module_deps)
        self.log.debug("List of toolchain elements from toolchain definition: %s" % toolchain_elements_mod_names)

        if toolchain_module_deps == toolchain_elements_mod_names:
            self.log.info("List of toolchain dependency modules and toolchain definition match!")
        else:
            self.log.error(
                "List of toolchain dependency modules and toolchain definition do not match "
                "(%s vs %s)" % (toolchain_module_deps, toolchain_elements_mod_names)
            )

        # Generate the variables to be set
        self.set_variables()

        # set the variables
        # onlymod can be comma-separated string of variables not to be set
        if onlymod == True:
            self.log.debug("prepare: do not set additional variables onlymod=%s" % onlymod)
            self.generate_vars()
        else:
            self.log.debug("prepare: set additional variables onlymod=%s" % onlymod)

            # add LDFLAGS and CPPFLAGS from dependencies to self.vars
            self._add_dependency_variables()
            self.generate_vars()
            self._setenv_variables(onlymod)