示例#1
0
    def test_v20(self):
        """Test parsing of easyconfig in format v2."""
        # hard enable experimental
        orig_experimental = easybuild.tools.build_log.EXPERIMENTAL
        easybuild.tools.build_log.EXPERIMENTAL = True

        fn = os.path.join(TESTDIRBASE, 'v2.0', 'GCC.eb')
        ecp = EasyConfigParser(fn)

        formatter = ecp._formatter
        self.assertEqual(formatter.VERSION, EasyVersion('2.0'))

        self.assertTrue('name' in formatter.pyheader_localvars)
        self.assertFalse('version' in formatter.pyheader_localvars)
        self.assertFalse('toolchain' in formatter.pyheader_localvars)

        # this should be ok: ie the default values
        ec = ecp.get_config_dict()
        self.assertEqual(ec['toolchain'], {
            'name': 'system',
            'version': 'system'
        })
        self.assertEqual(ec['name'], 'GCC')
        self.assertEqual(ec['version'], '4.6.2')

        # changes to this dict should not affect the return value of the next call to get_config_dict
        fn = 'test.tar.gz'
        ec['sources'].append(fn)

        ec_bis = ecp.get_config_dict()
        self.assertTrue(fn in ec['sources'])
        self.assertFalse(fn in ec_bis['sources'])

        # restore
        easybuild.tools.build_log.EXPERIMENTAL = orig_experimental
    def test_v20(self):
        """Test parsing of easyconfig in format v2."""
        # hard enable experimental
        orig_experimental = easybuild.tools.build_log.EXPERIMENTAL
        easybuild.tools.build_log.EXPERIMENTAL = True

        fn = os.path.join(TESTDIRBASE, 'v2.0', 'GCC.eb')
        ecp = EasyConfigParser(fn)

        formatter = ecp._formatter
        self.assertEqual(formatter.VERSION, EasyVersion('2.0'))

        self.assertTrue('name' in formatter.pyheader_localvars)
        self.assertFalse('version' in formatter.pyheader_localvars)
        self.assertFalse('toolchain' in formatter.pyheader_localvars)

        # this should be ok: ie the default values
        ec = ecp.get_config_dict()
        self.assertEqual(ec['toolchain'], {
            'name': 'dummy',
            'version': 'dummy'
        })
        self.assertEqual(ec['name'], 'GCC')
        self.assertEqual(ec['version'], '4.6.2')

        # restore
        easybuild.tools.build_log.EXPERIMENTAL = orig_experimental
示例#3
0
    def test_tweak_one_version(self):
        """Test tweak_one function"""
        test_easyconfigs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
        toy_ec = os.path.join(test_easyconfigs_path, 't', 'toy', 'toy-0.0.eb')

        # test tweaking of software version (--try-software-version)
        tweaked_toy_ec = os.path.join(self.test_prefix, 'toy-tweaked.eb')
        tweak_one(toy_ec, tweaked_toy_ec, {'version': '1.2.3'})

        toy_ec_parsed = EasyConfigParser(toy_ec).get_config_dict()
        tweaked_toy_ec_parsed = EasyConfigParser(tweaked_toy_ec).get_config_dict()

        # checksums should be reset to empty list, only version should be changed, nothing else
        self.assertEqual(tweaked_toy_ec_parsed['checksums'], [])
        self.assertEqual(tweaked_toy_ec_parsed['version'], '1.2.3')
        for key in [k for k in toy_ec_parsed.keys() if k not in ['checksums', 'version']]:
            val = toy_ec_parsed[key]
            self.assertTrue(key in tweaked_toy_ec_parsed, "Parameter '%s' not defined in tweaked easyconfig file" % key)
            tweaked_val = tweaked_toy_ec_parsed.get(key)
            self.assertEqual(val, tweaked_val, "Different value for %s parameter: %s vs %s" % (key, val, tweaked_val))

        # check behaviour if target file already exists
        error_pattern = "File exists, not overwriting it without --force"
        self.assertErrorRegex(EasyBuildError, error_pattern, tweak_one, toy_ec, tweaked_toy_ec, {'version': '1.2.3'})

        # existing file does get overwritten when --force is used
        init_config(build_options={'force': True, 'silent': True})
        write_file(tweaked_toy_ec, '')
        tweak_one(toy_ec, tweaked_toy_ec, {'version': '1.2.3'})
        tweaked_toy_ec_parsed = EasyConfigParser(tweaked_toy_ec).get_config_dict()
        self.assertEqual(tweaked_toy_ec_parsed['version'], '1.2.3')
    def parse(self):
        """
        Parse the file and set options
        mandatory requirements are checked here
        """
        if self.build_specs is None:
            arg_specs = {}
        elif isinstance(self.build_specs, dict):
            # build a new dictionary with only the expected keys, to pass as named arguments to get_config_dict()
            arg_specs = self.build_specs
        else:
            self.log.error("Specifications should be specified using a dictionary, got %s" % type(self.build_specs))
        self.log.debug("Obtained specs dict %s" % arg_specs)

        self.log.info("Parsing easyconfig file %s with rawcontent: %s" % (self.path, self.rawtxt))
        parser = EasyConfigParser(filename=self.path, rawcontent=self.rawtxt)
        parser.set_specifications(arg_specs)
        local_vars = parser.get_config_dict()
        self.log.debug("Parsed easyconfig as a dictionary: %s" % local_vars)

        # make sure all mandatory parameters are defined
        # this includes both generic mandatory parameters and software-specific parameters defined via extra_options
        missing_mandatory_keys = [key for key in self.mandatory if key not in local_vars]
        if missing_mandatory_keys:
            self.log.error("mandatory parameters not provided in %s: %s" % (self.path, missing_mandatory_keys))

        # provide suggestions for typos
        possible_typos = [(key, difflib.get_close_matches(key.lower(), self._config.keys(), 1, 0.85))
                          for key in local_vars if key not in self]

        typos = [(key, guesses[0]) for (key, guesses) in possible_typos if len(guesses) == 1]
        if typos:
            self.log.error("You may have some typos in your easyconfig file: %s" %
                            ', '.join(["%s -> %s" % typo for typo in typos]))

        # we need toolchain to be set when we call _parse_dependency
        for key in ['toolchain'] + local_vars.keys():
            # validations are skipped, just set in the config
            # do not store variables we don't need
            if key in self._config.keys():
                if key in ['builddependencies', 'dependencies']:
                    self[key] = [self._parse_dependency(dep) for dep in local_vars[key]]
                elif key in ['hiddendependencies']:
                    self[key] = [self._parse_dependency(dep, hidden=True) for dep in local_vars[key]]
                else:
                    self[key] = local_vars[key]
                tup = (key, self[key], type(self[key]))
                self.log.info("setting config option %s: value %s (type: %s)" % tup)
            elif key in REPLACED_PARAMETERS:
                _log.nosupport("Easyconfig parameter '%s' is replaced by '%s'" % (key, REPLACED_PARAMETERS[key]), '2.0')

            else:
                self.log.debug("Ignoring unknown config option %s (value: %s)" % (key, local_vars[key]))

        # update templating dictionary
        self.generate_template_values()

        # indicate that this is a parsed easyconfig
        self._config['parsed'] = [True, "This is a parsed easyconfig", "HIDDEN"]
    def parse(self):
        """
        Parse the file and set options
        mandatory requirements are checked here
        """
        if self.build_specs is None:
            arg_specs = {}
        elif isinstance(self.build_specs, dict):
            # build a new dictionary with only the expected keys, to pass as named arguments to get_config_dict()
            arg_specs = self.build_specs
        else:
            self.log.error("Specifications should be specified using a dictionary, got %s" % type(self.build_specs))
        self.log.debug("Obtained specs dict %s" % arg_specs)

        parser = EasyConfigParser(self.path)
        parser.set_specifications(arg_specs)
        local_vars = parser.get_config_dict()
        self.log.debug("Parsed easyconfig as a dictionary: %s" % local_vars)

        # validate mandatory keys
        # TODO: remove this code. this is now (also) checked in the format (see validate_pyheader)
        missing_keys = [key for key in self.mandatory if key not in local_vars]
        if missing_keys:
            self.log.error("mandatory variables %s not provided in %s" % (missing_keys, self.path))

        # provide suggestions for typos
        possible_typos = [(key, difflib.get_close_matches(key.lower(), self._config.keys(), 1, 0.85))
                          for key in local_vars if key not in self._config]

        typos = [(key, guesses[0]) for (key, guesses) in possible_typos if len(guesses) == 1]
        if typos:
            self.log.error("You may have some typos in your easyconfig file: %s" %
                            ', '.join(["%s -> %s" % typo for typo in typos]))

        # we need toolchain to be set when we call _parse_dependency
        for key in ['toolchain'] + local_vars.keys():
            # validations are skipped, just set in the config
            # do not store variables we don't need
            if key in self._config.keys() + DEPRECATED_OPTIONS.keys():
                if key in ['builddependencies', 'dependencies']:
                    self[key] = [self._parse_dependency(dep) for dep in local_vars[key]]
                elif key in ['hiddendependencies']:
                    self[key] = [self._parse_dependency(dep, hidden=True) for dep in local_vars[key]]
                else:
                    self[key] = local_vars[key]
                tup = (key, self[key], type(self[key]))
                self.log.info("setting config option %s: value %s (type: %s)" % tup)

            else:
                self.log.debug("Ignoring unknown config option %s (value: %s)" % (key, local_vars[key]))

        # update templating dictionary
        self.generate_template_values()

        # indicate that this is a parsed easyconfig
        self._config['parsed'] = [True, "This is a parsed easyconfig", "HIDDEN"]
    def test_v10(self):
        ecp = EasyConfigParser(os.path.join(TESTDIRBASE, 'v1.0', 'GCC-4.6.3.eb'))

        self.assertEqual(ecp._formatter.VERSION, EasyVersion('1.0'))

        ec = ecp.get_config_dict()

        self.assertEqual(ec['toolchain'], {'name': 'dummy', 'version': 'dummy'})
        self.assertEqual(ec['name'], 'GCC')
        self.assertEqual(ec['version'], '4.6.3')
示例#7
0
    def test_v10(self):
        ecp = EasyConfigParser(os.path.join(TESTDIRBASE, 'v1.0', 'GCC-4.6.3.eb'))

        self.assertEqual(ecp._formatter.VERSION, EasyVersion('1.0'))

        ec = ecp.get_config_dict()

        self.assertEqual(ec['toolchain'], {'name': 'dummy', 'version': 'dummy'})
        self.assertEqual(ec['name'], 'GCC')
        self.assertEqual(ec['version'], '4.6.3')
    def test_v10(self):
        ecp = EasyConfigParser(os.path.join(TESTDIRBASE, "v1.0", "GCC-4.6.3.eb"))

        self.assertEqual(ecp._formatter.VERSION, EasyVersion("1.0"))

        ec = ecp.get_config_dict()

        self.assertEqual(ec["toolchain"], {"name": "dummy", "version": "dummy"})
        self.assertEqual(ec["name"], "GCC")
        self.assertEqual(ec["version"], "4.6.3")
    def test_check_value_types(self):
        """Test checking of easyconfig parameter value types."""
        test_ec = os.path.join(TESTDIRBASE, 'test_ecs', 'g', 'gzip', 'gzip-1.4-broken.eb')
        error_msg_pattern = "Type checking of easyconfig parameter values failed: .*'version'.*"
        ecp = EasyConfigParser(test_ec, auto_convert_value_types=False)
        self.assertErrorRegex(EasyBuildError, error_msg_pattern, ecp.get_config_dict)

        # test default behaviour: auto-converting of mismatched value types
        ecp = EasyConfigParser(test_ec)
        ecdict = ecp.get_config_dict()
        self.assertEqual(ecdict['version'], '1.4')
    def parse(self):
        """
        Parse the file and set options
        mandatory requirements are checked here
        """
        if self.build_specs is None:
            arg_specs = {}
        elif isinstance(self.build_specs, dict):
            # build a new dictionary with only the expected keys, to pass as named arguments to get_config_dict()
            arg_specs = self.build_specs
        else:
            self.log.error("Specifications should be specified using a dictionary, got %s" % type(self.build_specs))
        self.log.debug("Obtained specs dict %s" % arg_specs)

        parser = EasyConfigParser(self.path)
        parser.set_specifications(arg_specs)
        local_vars = parser.get_config_dict()
        self.log.debug("Parsed easyconfig as a dictionary: %s" % local_vars)

        # validate mandatory keys
        # TODO: remove this code. this is now (also) checked in the format (see validate_pyheader)
        missing_keys = [key for key in self.mandatory if key not in local_vars]
        if missing_keys:
            self.log.error("mandatory variables %s not provided in %s" % (missing_keys, self.path))

        # provide suggestions for typos
        possible_typos = [(key, difflib.get_close_matches(key.lower(), self._config.keys(), 1, 0.85))
                          for key in local_vars if key not in self._config]

        typos = [(key, guesses[0]) for (key, guesses) in possible_typos if len(guesses) == 1]
        if typos:
            self.log.error("You may have some typos in your easyconfig file: %s" %
                            ', '.join(["%s -> %s" % typo for typo in typos]))

        # we need toolchain to be set when we call _parse_dependency
        for key in ['toolchain'] + local_vars.keys():
            # validations are skipped, just set in the config
            # do not store variables we don't need
            if key in self._config.keys() + DEPRECATED_OPTIONS.keys():
                if key in ['builddependencies', 'dependencies']:
                    self[key] = [self._parse_dependency(dep) for dep in local_vars[key]]
                else:
                    self[key] = local_vars[key]
                tup = (key, self[key], type(self[key]))
                self.log.info("setting config option %s: value %s (type: %s)" % tup)

            else:
                self.log.debug("Ignoring unknown config option %s (value: %s)" % (key, local_vars[key]))

        # update templating dictionary
        self.generate_template_values()

        # indicate that this is a parsed easyconfig
        self._config['parsed'] = [True, "This is a parsed easyconfig", "HIDDEN"]
示例#11
0
    def test_check_value_types(self):
        """Test checking of easyconfig parameter value types."""
        test_ec = os.path.join(TESTDIRBASE, 'test_ecs', 'g', 'gzip', 'gzip-1.4-broken.eb')
        error_msg_pattern = "Type checking of easyconfig parameter values failed: .*'version'.*"
        ecp = EasyConfigParser(test_ec, auto_convert_value_types=False)
        self.assertErrorRegex(EasyBuildError, error_msg_pattern, ecp.get_config_dict)

        # test default behaviour: auto-converting of mismatched value types
        ecp = EasyConfigParser(test_ec)
        ecdict = ecp.get_config_dict()
        self.assertEqual(ecdict['version'], '1.4')
示例#12
0
    def parse(self, path, format_version=None):
        """
        Parse the file and set options
        mandatory requirements are checked here
        """
        parser = EasyConfigParser(path, format_version=format_version)
        local_vars = parser.get_config_dict()

        # validate mandatory keys
        # TODO: remove this code. this is now (also) checked in the format (see validate_pyheader)
        missing_keys = [key for key in self.mandatory if key not in local_vars]
        if missing_keys:
            self.log.error("mandatory variables %s not provided in %s" %
                           (missing_keys, path))

        # provide suggestions for typos
        possible_typos = [(key,
                           difflib.get_close_matches(key.lower(),
                                                     self._config.keys(), 1,
                                                     0.85))
                          for key in local_vars if key not in self._config]

        typos = [(key, guesses[0]) for (key, guesses) in possible_typos
                 if len(guesses) == 1]
        if typos:
            self.log.error(
                "You may have some typos in your easyconfig file: %s" %
                ', '.join(["%s -> %s" % typo for typo in typos]))

        self._legacy_license(local_vars)

        for key in local_vars:
            # validations are skipped, just set in the config
            # do not store variables we don't need
            if key in self._config:
                self[key] = local_vars[key]
                self.log.info("setting config option %s: value %s" %
                              (key, self[key]))

            else:
                self.log.debug(
                    "Ignoring unknown config option %s (value: %s)" %
                    (key, local_vars[key]))

        # update templating dictionary
        self.generate_template_values()

        # indicate that this is a parsed easyconfig
        self._config['parsed'] = [
            True, "This is a parsed easyconfig", "HIDDEN"
        ]
    def test_tweak_one_version(self):
        """Test tweak_one function"""
        test_easyconfigs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
        toy_ec = os.path.join(test_easyconfigs_path, 't', 'toy', 'toy-0.0.eb')

        # test tweaking of software version (--try-software-version)
        tweaked_toy_ec = os.path.join(self.test_prefix, 'toy-tweaked.eb')
        tweak_one(toy_ec, tweaked_toy_ec, {'version': '1.2.3'})

        toy_ec_parsed = EasyConfigParser(toy_ec).get_config_dict()
        tweaked_toy_ec_parsed = EasyConfigParser(tweaked_toy_ec).get_config_dict()

        # checksums should be reset to empty list, only version should be changed, nothing else
        self.assertEqual(tweaked_toy_ec_parsed['checksums'], [])
        self.assertEqual(tweaked_toy_ec_parsed['version'], '1.2.3')
        for key in [k for k in toy_ec_parsed.keys() if k not in ['checksums', 'version']]:
            val = toy_ec_parsed[key]
            self.assertTrue(key in tweaked_toy_ec_parsed, "Parameter '%s' not defined in tweaked easyconfig file" % key)
            tweaked_val = tweaked_toy_ec_parsed.get(key)
            self.assertEqual(val, tweaked_val, "Different value for %s parameter: %s vs %s" % (key, val, tweaked_val))

        # check behaviour if target file already exists
        error_pattern = "File exists, not overwriting it without --force"
        self.assertErrorRegex(EasyBuildError, error_pattern, tweak_one, toy_ec, tweaked_toy_ec, {'version': '1.2.3'})

        # existing file does get overwritten when --force is used
        init_config(build_options={'force': True, 'silent': True})
        write_file(tweaked_toy_ec, '')
        tweak_one(toy_ec, tweaked_toy_ec, {'version': '1.2.3'})
        tweaked_toy_ec_parsed = EasyConfigParser(tweaked_toy_ec).get_config_dict()
        self.assertEqual(tweaked_toy_ec_parsed['version'], '1.2.3')
    def test_v20_deps(self):
        """Test parsing of easyconfig in format v2 that includes dependencies."""
        # hard enable experimental
        orig_experimental = easybuild.tools.build_log.EXPERIMENTAL
        easybuild.tools.build_log.EXPERIMENTAL = True

        fn = os.path.join(TESTDIRBASE, 'v2.0', 'libpng.eb')
        ecp = EasyConfigParser(fn)

        ec = ecp.get_config_dict()
        self.assertEqual(ec['name'], 'libpng')
        # first version/toolchain listed is default
        self.assertEqual(ec['version'], '1.5.10')
        self.assertEqual(ec['toolchain'], {'name': 'goolf', 'version': '1.4.10'})

        # dependencies should be parsed correctly
        deps = ec['dependencies']
        self.assertTrue(isinstance(deps[0], Dependency))
        self.assertEqual(deps[0].name(), 'zlib')
        self.assertEqual(deps[0].version(), '1.2.5')

        fn = os.path.join(TESTDIRBASE, 'v2.0', 'goolf.eb')
        ecp = EasyConfigParser(fn)

        ec = ecp.get_config_dict()
        self.assertEqual(ec['name'], 'goolf')
        self.assertEqual(ec['version'], '1.4.10')
        self.assertEqual(ec['toolchain'], {'name': 'dummy', 'version': 'dummy'})

        # dependencies should be parsed correctly
        deps = [
            # name, version, versionsuffix, toolchain
            ('GCC', '4.7.2', None, None),
            ('OpenMPI', '1.6.4', None, {'name': 'GCC', 'version': '4.7.2'}),
            ('OpenBLAS', '0.2.6', '-LAPACK-3.4.2', {'name': 'gompi', 'version': '1.4.10'}),
            ('FFTW', '3.3.3', None, {'name': 'gompi', 'version': '1.4.10'}),
            ('ScaLAPACK', '2.0.2', '-OpenBLAS-0.2.6-LAPACK-3.4.2', {'name': 'gompi', 'version': '1.4.10'}),
        ]
        for i, (name, version, versionsuffix, toolchain) in enumerate(deps):
            self.assertEqual(ec['dependencies'][i].name(), name)
            self.assertEqual(ec['dependencies'][i].version(), version)
            self.assertEqual(ec['dependencies'][i].versionsuffix(), versionsuffix)
            self.assertEqual(ec['dependencies'][i].toolchain(), toolchain)

        # restore
        easybuild.tools.build_log.EXPERIMENTAL = orig_experimental
示例#15
0
 def check_ec(path, expected_buildstats):
     """Check easyconfig at specified path"""
     self.assertTrue(os.path.exists(path))
     ectxt = read_file(path)
     self.assertTrue(ectxt.startswith("# Built with EasyBuild version"))
     self.assertTrue("# Build statistics" in ectxt)
     ecdict = EasyConfigParser(path).get_config_dict()
     self.assertEqual(ecdict['buildstats'], expected_buildstats)
    def test_v20_deps(self):
        """Test parsing of easyconfig in format v2 that includes dependencies."""
        # hard enable experimental
        orig_experimental = easybuild.tools.build_log.EXPERIMENTAL
        easybuild.tools.build_log.EXPERIMENTAL = True

        fn = os.path.join(TESTDIRBASE, "v2.0", "libpng.eb")
        ecp = EasyConfigParser(fn)

        ec = ecp.get_config_dict()
        self.assertEqual(ec["name"], "libpng")
        # first version/toolchain listed is default
        self.assertEqual(ec["version"], "1.5.10")
        self.assertEqual(ec["toolchain"], {"name": "goolf", "version": "1.4.10"})

        # dependencies should be parsed correctly
        deps = ec["dependencies"]
        self.assertTrue(isinstance(deps[0], Dependency))
        self.assertEqual(deps[0].name(), "zlib")
        self.assertEqual(deps[0].version(), "1.2.5")

        fn = os.path.join(TESTDIRBASE, "v2.0", "goolf.eb")
        ecp = EasyConfigParser(fn)

        ec = ecp.get_config_dict()
        self.assertEqual(ec["name"], "goolf")
        self.assertEqual(ec["version"], "1.4.10")
        self.assertEqual(ec["toolchain"], {"name": "dummy", "version": "dummy"})

        # dependencies should be parsed correctly
        deps = [
            # name, version, versionsuffix, toolchain
            ("GCC", "4.7.2", None, None),
            ("OpenMPI", "1.6.4", None, {"name": "GCC", "version": "4.7.2"}),
            ("OpenBLAS", "0.2.6", "-LAPACK-3.4.2", {"name": "gompi", "version": "1.4.10"}),
            ("FFTW", "3.3.3", None, {"name": "gompi", "version": "1.4.10"}),
            ("ScaLAPACK", "2.0.2", "-OpenBLAS-0.2.6-LAPACK-3.4.2", {"name": "gompi", "version": "1.4.10"}),
        ]
        for i, (name, version, versionsuffix, toolchain) in enumerate(deps):
            self.assertEqual(ec["dependencies"][i].name(), name)
            self.assertEqual(ec["dependencies"][i].version(), version)
            self.assertEqual(ec["dependencies"][i].versionsuffix(), versionsuffix)
            self.assertEqual(ec["dependencies"][i].toolchain(), toolchain)

        # restore
        easybuild.tools.build_log.EXPERIMENTAL = orig_experimental
    def test_raw(self):
        """Test passing of raw contents to EasyConfigParser."""
        ec_file1 = os.path.join(TESTDIRBASE, 'v1.0', 'g', 'GCC',
                                'GCC-4.6.3.eb')
        ec_txt1 = read_file(ec_file1)
        ec_file2 = os.path.join(TESTDIRBASE, 'v1.0', 'g', 'gzip',
                                'gzip-1.5-goolf-1.4.10.eb')
        ec_txt2 = read_file(ec_file2)

        ecparser = EasyConfigParser(ec_file1)
        self.assertEqual(ecparser.rawcontent, ec_txt1)

        ecparser = EasyConfigParser(rawcontent=ec_txt2)
        self.assertEqual(ecparser.rawcontent, ec_txt2)

        # rawcontent supersedes passed filepath
        ecparser = EasyConfigParser(ec_file1, rawcontent=ec_txt2)
        self.assertEqual(ecparser.rawcontent, ec_txt2)
        ec = ecparser.get_config_dict()
        self.assertEqual(ec['name'], 'gzip')
        self.assertEqual(ec['toolchain']['name'], 'goolf')

        self.assertErrorRegex(EasyBuildError,
                              "Neither filename nor rawcontent provided",
                              EasyConfigParser)
def doit(path, pattern):
    """Do it for all easyconfigs in specified location"""
    repo = git.Repo.init(path)

    timestamp_regex = re.compile('Date:\s*(?P<timestamp>.*)', re.M)
    tc_regex = re.compile(r"^\s*toolchain\s*=\s*(.*)$", re.M)

    easyconfigs, toolchains = [], set()

    specs = []
    for (subpath, _, filenames) in os.walk(path, topdown=True):
        if subpath.startswith(os.path.join(path, '.git')):
            continue

        for filename in filenames:
            if filename.endswith(
                    '.eb'
            ) and filename != 'TEMPLATE.eb' and pattern in filename:

                specs.append(os.path.join(path, subpath, filename))

    for idx, spec in enumerate(specs):
        print "\rprocessed %d of %d easyconfigs" % (idx, len(specs)),

        ec = EasyConfigParser(filename=spec).get_config_dict()

        if ec['toolchain']['name'] == DUMMY_TOOLCHAIN_NAME:
            toolchain = 'dummy'
        else:
            toolchain = '%(name)s-%(version)s' % ec['toolchain']
        toolchains.add(toolchain)

        logtxt = repo.git.log('--reverse', '--date=iso-local', spec)
        res = timestamp_regex.search(logtxt)
        if res:
            timestamp = str(res.group('timestamp'))
        else:
            raise EasyBuildError("No timestamp found in git log for %s: %s",
                                 spec, logtxt)

        easyconfigs.append((timestamp, os.path.basename(spec), toolchain))

    print ''
    print 'found %d different toolchains' % len(toolchains)

    for toolchain in sorted(toolchains):
        # datestamp first for correct sorting!
        ecs = sorted([(datestamp, ec) for (datestamp, ec, tc) in easyconfigs
                      if tc == toolchain])

        print '%s (%d)' % (toolchain, len(ecs))
        print '\toldest: %s (%s)' % (ecs[0][1], ecs[0][0])
        print '\tnewest: %s (%s)' % (ecs[-1][1], ecs[-1][0])
    def test_v20_extra(self):
        fn = os.path.join(TESTDIRBASE, 'v2.0', 'doesnotexist.eb')
        ecp = EasyConfigParser(fn)

        formatter = ecp._formatter
        self.assertEqual(formatter.VERSION, EasyVersion('2.0'))

        self.assertTrue('name' in formatter.pyheader_localvars)
        self.assertFalse('version' in formatter.pyheader_localvars)
        self.assertFalse('toolchain' in formatter.pyheader_localvars)

        self.assertRaises(NotImplementedError, ecp.get_config_dict)
    def test_raw(self):
        """Test passing of raw contents to EasyConfigParser."""
        ec_file1 = os.path.join(TESTDIRBASE, 'v1.0', 'g', 'GCC', 'GCC-4.6.3.eb')
        ec_txt1 = read_file(ec_file1)
        ec_file2 = os.path.join(TESTDIRBASE, 'v1.0', 'g', 'gzip', 'gzip-1.5-goolf-1.4.10.eb')
        ec_txt2 = read_file(ec_file2)

        ecparser = EasyConfigParser(ec_file1)
        self.assertEqual(ecparser.rawcontent, ec_txt1)

        ecparser = EasyConfigParser(rawcontent=ec_txt2)
        self.assertEqual(ecparser.rawcontent, ec_txt2)

        # rawcontent supersedes passed filepath
        ecparser = EasyConfigParser(ec_file1, rawcontent=ec_txt2)
        self.assertEqual(ecparser.rawcontent, ec_txt2)
        ec = ecparser.get_config_dict()
        self.assertEqual(ec['name'], 'gzip')
        self.assertEqual(ec['toolchain']['name'], 'goolf')

        self.assertErrorRegex(EasyBuildError, "Neither filename nor rawcontent provided", EasyConfigParser)
示例#21
0
    def parse(self, path, format_version=None):
        """
        Parse the file and set options
        mandatory requirements are checked here
        """
        parser = EasyConfigParser(path, format_version=format_version)
        local_vars = parser.get_config_dict()

        # validate mandatory keys
        # TODO: remove this code. this is now (also) checked in the format (see validate_pyheader)
        missing_keys = [key for key in self.mandatory if key not in local_vars]
        if missing_keys:
            self.log.error("mandatory variables %s not provided in %s" % (missing_keys, path))

        # provide suggestions for typos
        possible_typos = [(key, difflib.get_close_matches(key.lower(), self._config.keys(), 1, 0.85))
                          for key in local_vars if key not in self._config]

        typos = [(key, guesses[0]) for (key, guesses) in possible_typos if len(guesses) == 1]
        if typos:
            self.log.error("You may have some typos in your easyconfig file: %s" %
                            ', '.join(["%s -> %s" % typo for typo in typos]))

        self._legacy_license(local_vars)

        for key in local_vars:
            # validations are skipped, just set in the config
            # do not store variables we don't need
            if key in self._config:
                self[key] = local_vars[key]
                self.log.info("setting config option %s: value %s" % (key, self[key]))

            else:
                self.log.debug("Ignoring unknown config option %s (value: %s)" % (key, local_vars[key]))

        # update templating dictionary
        self.generate_template_values()

        # indicate that this is a parsed easyconfig
        self._config['parsed'] = [True, "This is a parsed easyconfig", "HIDDEN"]
示例#22
0
    def test_tweak_one_version(self):
        """Test tweak_one function"""
        test_easyconfigs_path = os.path.join(
            os.path.dirname(os.path.abspath(__file__)), 'easyconfigs',
            'test_ecs')
        toy_ec = os.path.join(test_easyconfigs_path, 't', 'toy', 'toy-0.0.eb')

        # test tweaking of software version (--try-software-version)
        tweaked_toy_ec = os.path.join(self.test_prefix, 'toy-tweaked.eb')
        tweak_one(toy_ec, tweaked_toy_ec, {'version': '1.2.3'})

        toy_ec_parsed = EasyConfigParser(toy_ec).get_config_dict()
        tweaked_toy_ec_parsed = EasyConfigParser(
            tweaked_toy_ec).get_config_dict()

        # checksums should be reset to empty list, only version should be changed, nothing else
        self.assertEqual(tweaked_toy_ec_parsed['checksums'], [])
        self.assertEqual(tweaked_toy_ec_parsed['version'], '1.2.3')
        for key in [
                k for k in toy_ec_parsed.keys()
                if k not in ['checksums', 'version']
        ]:
            val = toy_ec_parsed[key]
            self.assertTrue(
                key in tweaked_toy_ec_parsed,
                "Parameter '%s' not defined in tweaked easyconfig file" % key)
            tweaked_val = tweaked_toy_ec_parsed.get(key)
            self.assertEqual(
                val, tweaked_val,
                "Different value for %s parameter: %s vs %s" %
                (key, val, tweaked_val))
示例#23
0
    def test_v10(self):
        ecp = EasyConfigParser(
            os.path.join(TESTDIRBASE, 'v1.0', 'g', 'GCC', 'GCC-4.6.3.eb'))

        self.assertEqual(ecp._formatter.VERSION, EasyVersion('1.0'))

        ec = ecp.get_config_dict()

        self.assertEqual(ec['toolchain'], {
            'name': 'system',
            'version': 'system'
        })
        self.assertEqual(ec['name'], 'GCC')
        self.assertEqual(ec['version'], '4.6.3')

        # changes to this dict should not affect the return value of the next call to get_config_dict
        fn = 'test.tar.gz'
        ec['sources'].append(fn)

        ec_bis = ecp.get_config_dict()
        self.assertTrue(fn in ec['sources'])
        self.assertFalse(fn in ec_bis['sources'])
    def test_v20(self):
        """Test parsing of easyconfig in format v2."""
        # hard enable experimental
        orig_experimental = easybuild.tools.build_log.EXPERIMENTAL
        easybuild.tools.build_log.EXPERIMENTAL = True

        fn = os.path.join(TESTDIRBASE, "v2.0", "GCC.eb")
        ecp = EasyConfigParser(fn)

        formatter = ecp._formatter
        self.assertEqual(formatter.VERSION, EasyVersion("2.0"))

        self.assertTrue("name" in formatter.pyheader_localvars)
        self.assertFalse("version" in formatter.pyheader_localvars)
        self.assertFalse("toolchain" in formatter.pyheader_localvars)

        # this should be ok: ie the default values
        ec = ecp.get_config_dict()
        self.assertEqual(ec["toolchain"], {"name": "dummy", "version": "dummy"})
        self.assertEqual(ec["name"], "GCC")
        self.assertEqual(ec["version"], "4.6.2")

        # restore
        easybuild.tools.build_log.EXPERIMENTAL = orig_experimental
    def test_v20(self):
        """Test parsing of easyconfig in format v2."""
        # hard enable experimental
        orig_experimental = easybuild.tools.build_log.EXPERIMENTAL
        easybuild.tools.build_log.EXPERIMENTAL = True

        fn = os.path.join(TESTDIRBASE, 'v2.0', 'GCC.eb')
        ecp = EasyConfigParser(fn)

        formatter = ecp._formatter
        self.assertEqual(formatter.VERSION, EasyVersion('2.0'))

        self.assertTrue('name' in formatter.pyheader_localvars)
        self.assertFalse('version' in formatter.pyheader_localvars)
        self.assertFalse('toolchain' in formatter.pyheader_localvars)

        # this should be ok: ie the default values
        ec = ecp.get_config_dict()
        self.assertEqual(ec['toolchain'], {'name': 'dummy', 'version': 'dummy'})
        self.assertEqual(ec['name'], 'GCC')
        self.assertEqual(ec['version'], '4.6.2')

        # restore
        easybuild.tools.build_log.EXPERIMENTAL = orig_experimental
示例#26
0
    def test_v20_extra(self):
        """Test parsing of easyconfig in format v2."""
        # hard enable experimental
        orig_experimental = easybuild.tools.build_log.EXPERIMENTAL
        easybuild.tools.build_log.EXPERIMENTAL = True

        fn = os.path.join(TESTDIRBASE, 'v2.0', 'doesnotexist.eb')
        ecp = EasyConfigParser(fn)

        formatter = ecp._formatter
        self.assertEqual(formatter.VERSION, EasyVersion('2.0'))

        self.assertTrue('name' in formatter.pyheader_localvars)
        self.assertFalse('version' in formatter.pyheader_localvars)
        self.assertFalse('toolchain' in formatter.pyheader_localvars)

        # restore
        easybuild.tools.build_log.EXPERIMENTAL = orig_experimental
示例#27
0
    def test_v20_deps(self):
        """Test parsing of easyconfig in format v2 that includes dependencies."""
        # hard enable experimental
        orig_experimental = easybuild.tools.build_log.EXPERIMENTAL
        easybuild.tools.build_log.EXPERIMENTAL = True

        fn = os.path.join(TESTDIRBASE, 'v2.0', 'libpng.eb')
        ecp = EasyConfigParser(fn)

        ec = ecp.get_config_dict()
        self.assertEqual(ec['name'], 'libpng')
        # first version/toolchain listed is default
        self.assertEqual(ec['version'], '1.5.10')
        self.assertEqual(ec['toolchain'], {'name': 'goolf', 'version': '1.4.10'})

        # dependencies should be parsed correctly
        deps = ec['dependencies']
        self.assertTrue(isinstance(deps[0], Dependency))
        self.assertEqual(deps[0].name(), 'zlib')
        self.assertEqual(deps[0].version(), '1.2.5')

        fn = os.path.join(TESTDIRBASE, 'v2.0', 'goolf.eb')
        ecp = EasyConfigParser(fn)

        ec = ecp.get_config_dict()
        self.assertEqual(ec['name'], 'goolf')
        self.assertEqual(ec['version'], '1.4.10')
        self.assertEqual(ec['toolchain'], {'name': 'dummy', 'version': 'dummy'})

        # dependencies should be parsed correctly
        deps = [
            # name, version, versionsuffix, toolchain
            ('GCC', '4.7.2', None, None),
            ('OpenMPI', '1.6.4', None, {'name': 'GCC', 'version': '4.7.2'}),
            ('OpenBLAS', '0.2.6', '-LAPACK-3.4.2', {'name': 'gompi', 'version': '1.4.10'}),
            ('FFTW', '3.3.3', None, {'name': 'gompi', 'version': '1.4.10'}),
            ('ScaLAPACK', '2.0.2', '-OpenBLAS-0.2.6-LAPACK-3.4.2', {'name': 'gompi', 'version': '1.4.10'}),
        ]
        for i, (name, version, versionsuffix, toolchain) in enumerate(deps):
            self.assertEqual(ec['dependencies'][i].name(), name)
            self.assertEqual(ec['dependencies'][i].version(), version)
            self.assertEqual(ec['dependencies'][i].versionsuffix(), versionsuffix)
            self.assertEqual(ec['dependencies'][i].toolchain(), toolchain)

        # restore
        easybuild.tools.build_log.EXPERIMENTAL = orig_experimental
示例#28
0
def template_easyconfig_test(self, spec):
    """Tests for an individual easyconfig: parsing, instantiating easyblock, check patches, ..."""

    # set to False, so it's False in case of this test failing
    global single_tests_ok
    prev_single_tests_ok = single_tests_ok
    single_tests_ok = False

    # parse easyconfig
    ecs = process_easyconfig(spec)
    if len(ecs) == 1:
        ec = ecs[0]['ec']

        # cache the parsed easyconfig, to avoid that it is parsed again
        self.parsed_easyconfigs.append(ecs[0])
    else:
        self.assertTrue(
            False,
            "easyconfig %s does not contain blocks, yields only one parsed easyconfig"
            % spec)

    # check easyconfig file name
    expected_fn = '%s-%s.eb' % (ec['name'], det_full_ec_version(ec))
    msg = "Filename '%s' of parsed easyconfig matches expected filename '%s'" % (
        spec, expected_fn)
    self.assertEqual(os.path.basename(spec), expected_fn, msg)

    name, easyblock = fetch_parameters_from_easyconfig(ec.rawtxt,
                                                       ['name', 'easyblock'])

    # make sure easyconfig file is in expected location
    expected_subdir = os.path.join('easybuild', 'easyconfigs',
                                   letter_dir_for(name), name)
    subdir = os.path.join(*spec.split(os.path.sep)[-5:-1])
    fail_msg = "Easyconfig file %s not in expected subdirectory %s" % (
        spec, expected_subdir)
    self.assertEqual(expected_subdir, subdir, fail_msg)

    # sanity check for software name, moduleclass
    self.assertEqual(ec['name'], name)
    self.assertTrue(ec['moduleclass'] in build_option('valid_module_classes'))

    # instantiate easyblock with easyconfig file
    app_class = get_easyblock_class(easyblock, name=name)

    # check that automagic fallback to ConfigureMake isn't done (deprecated behaviour)
    fn = os.path.basename(spec)
    error_msg = "%s relies on automagic fallback to ConfigureMake, should use easyblock = 'ConfigureMake' instead" % fn
    self.assertTrue(easyblock or app_class is not ConfigureMake, error_msg)

    app = app_class(ec)

    # more sanity checks
    self.assertTrue(name, app.name)
    self.assertTrue(ec['version'], app.version)

    # make sure that $root is not used, since it is not compatible with module files in Lua syntax
    res = re.findall('.*\$root.*', ec.rawtxt, re.M)
    error_msg = "Found use of '$root', not compatible with modules in Lua syntax, use '%%(installdir)s' instead: %s"
    self.assertFalse(res, error_msg % res)

    # make sure old GitHub urls for EasyBuild that include 'hpcugent' are no longer used
    old_urls = [
        'github.com/hpcugent/easybuild',
        'hpcugent.github.com/easybuild',
        'hpcugent.github.io/easybuild',
    ]
    for old_url in old_urls:
        self.assertFalse(old_url in ec.rawtxt,
                         "Old URL '%s' not found in %s" % (old_url, spec))

    # make sure binutils is included as a build dep if toolchain is GCCcore
    if ec['toolchain']['name'] == 'GCCcore':
        # with 'Tarball' easyblock: only unpacking, no building; Eigen is also just a tarball
        requires_binutils = ec['easyblock'] not in [
            'Tarball'
        ] and ec['name'] not in ['Eigen']

        # let's also exclude the very special case where the system GCC is used as GCCcore, and only apply this
        # exception to the dependencies of binutils (since we should eventually build a new binutils with GCCcore)
        if ec['toolchain']['version'] == 'system':
            binutils_complete_dependencies = [
                'M4', 'Bison', 'flex', 'help2man', 'zlib', 'binutils'
            ]
            requires_binutils &= bool(
                ec['name'] not in binutils_complete_dependencies)

        # if no sources/extensions/components are specified, it's just a bundle (nothing is being compiled)
        requires_binutils &= bool(ec['sources'] or ec['exts_list']
                                  or ec.get('components'))

        if requires_binutils:
            dep_names = [d['name'] for d in ec.builddependencies()]
            self.assertTrue(
                'binutils' in dep_names,
                "binutils is a build dep in %s: %s" % (spec, dep_names))

    # make sure all patch files are available
    specdir = os.path.dirname(spec)
    specfn = os.path.basename(spec)
    for patch in ec['patches']:
        if isinstance(patch, (tuple, list)):
            patch = patch[0]
        # only check actual patch files, not other files being copied via the patch functionality
        if patch.endswith('.patch'):
            patch_full = os.path.join(specdir, patch)
            msg = "Patch file %s is available for %s" % (patch_full, specfn)
            self.assertTrue(os.path.isfile(patch_full), msg)

    for ext in ec['exts_list']:
        if isinstance(ext, (tuple, list)) and len(ext) == 3:
            self.assertTrue(isinstance(ext[2], dict),
                            "3rd element of extension spec is a dictionary")
            for ext_patch in ext[2].get('patches', []):
                if isinstance(ext_patch, (tuple, list)):
                    ext_patch = ext_patch[0]
                # only check actual patch files, not other files being copied via the patch functionality
                if ext_patch.endswith('.patch'):
                    ext_patch_full = os.path.join(specdir, ext_patch)
                    msg = "Patch file %s is available for %s" % (
                        ext_patch_full, specfn)
                    self.assertTrue(os.path.isfile(ext_patch_full), msg)

    # check whether all extra_options defined for used easyblock are defined
    extra_opts = app.extra_options()
    for key in extra_opts:
        self.assertTrue(key in app.cfg)

    app.close_log()
    os.remove(app.logfile)

    # dump the easyconfig file
    handle, test_ecfile = tempfile.mkstemp()
    os.close(handle)

    ec.dump(test_ecfile)
    dumped_ec = EasyConfigParser(test_ecfile).get_config_dict()
    os.remove(test_ecfile)

    # inject dummy values for templates that are only known at a later stage
    dummy_template_values = {
        'builddir': '/dummy/builddir',
        'installdir': '/dummy/installdir',
    }
    ec.template_values.update(dummy_template_values)

    ec_dict = ec.parser.get_config_dict()
    orig_toolchain = ec_dict['toolchain']
    for key in ec_dict:
        # skip parameters for which value is equal to default value
        orig_val = ec_dict[key]
        if key in DEFAULT_CONFIG and orig_val == DEFAULT_CONFIG[key][0]:
            continue
        if key in extra_opts and orig_val == extra_opts[key][0]:
            continue
        if key not in DEFAULT_CONFIG and key not in extra_opts:
            continue

        orig_val = resolve_template(ec_dict[key], ec.template_values)
        dumped_val = resolve_template(dumped_ec[key], ec.template_values)

        # take into account that dumped value for *dependencies may include hard-coded subtoolchains
        # if no easyconfig was found for the dependency with the 'parent' toolchain,
        # if may get resolved using a subtoolchain, which is then hardcoded in the dumped easyconfig
        if key in DEPENDENCY_PARAMETERS:
            # number of dependencies should remain the same
            self.assertEqual(len(orig_val), len(dumped_val))
            for orig_dep, dumped_dep in zip(orig_val, dumped_val):
                # name/version should always match
                self.assertEqual(orig_dep[:2], dumped_dep[:2])

                # 3rd value is versionsuffix;
                if len(dumped_dep) >= 3:
                    # if no versionsuffix was specified in original dep spec, then dumped value should be empty string
                    if len(orig_dep) >= 3:
                        self.assertEqual(dumped_dep[2], orig_dep[2])
                    else:
                        self.assertEqual(dumped_dep[2], '')

                # 4th value is toolchain spec
                if len(dumped_dep) >= 4:
                    if len(orig_dep) >= 4:
                        self.assertEqual(dumped_dep[3], orig_dep[3])
                    else:
                        # if a subtoolchain is specifed (only) in the dumped easyconfig,
                        # it should *not* be the same as the parent toolchain
                        self.assertNotEqual(dumped_dep[3],
                                            (orig_toolchain['name'],
                                             orig_toolchain['version']))

        else:
            self.assertEqual(orig_val, dumped_val)

    # test passed, so set back to True
    single_tests_ok = True and prev_single_tests_ok
示例#29
0
def list_software(output_format=FORMAT_TXT,
                  detailed=False,
                  only_installed=False):
    """
    Show list of supported software

    :param output_format: output format to use
    :param detailed: whether or not to return detailed information (incl. version, versionsuffix, toolchain info)
    :param only_installed: only retain software for which a corresponding module is available
    :return: multi-line string presenting requested info
    """
    silent = build_option('silent')

    ec_paths = find_matching_easyconfigs('*', '*',
                                         build_option('robot_path') or [])
    ecs = []
    cnt = len(ec_paths)
    for idx, ec_path in enumerate(ec_paths):
        # full EasyConfig instance is only required when module name is needed
        # this is significantly slower (5-10x) than a 'shallow' parse via EasyConfigParser
        if only_installed:
            ec = process_easyconfig(ec_path, validate=False,
                                    parse_only=True)[0]['ec']
        else:
            ec = EasyConfigParser(filename=ec_path).get_config_dict()

        ecs.append(ec)
        print_msg('\r', prefix=False, newline=False, silent=silent)
        print_msg("Processed %d/%d easyconfigs..." % (idx + 1, cnt),
                  newline=False,
                  silent=silent)
    print_msg('', prefix=False, silent=silent)

    software = {}
    for ec in ecs:
        software.setdefault(ec['name'], [])
        if is_system_toolchain(ec['toolchain']['name']):
            toolchain = SYSTEM_TOOLCHAIN_NAME
        else:
            toolchain = '%s/%s' % (ec['toolchain']['name'],
                                   ec['toolchain']['version'])

        keys = ['description', 'homepage', 'version', 'versionsuffix']

        info = {'toolchain': toolchain}
        for key in keys:
            info[key] = ec.get(key, '')

        # make sure values like homepage & versionsuffix get properly templated
        if isinstance(ec, dict):
            template_values = template_constant_dict(ec)
            for key in keys:
                if '%(' in info[key]:
                    try:
                        info[key] = info[key] % template_values
                    except (KeyError, TypeError, ValueError) as err:
                        _log.debug("Ignoring failure to resolve templates: %s",
                                   err)

        software[ec['name']].append(info)

        if only_installed:
            software[ec['name']][-1].update({'mod_name': ec.full_mod_name})

    print_msg("Found %d different software packages" % len(software),
              silent=silent)

    if only_installed:
        avail_mod_names = modules_tool().available()

        # rebuild software, only retain entries with a corresponding available module
        software, all_software = {}, software
        for key in all_software:
            for entry in all_software[key]:
                if entry['mod_name'] in avail_mod_names:
                    software.setdefault(key, []).append(entry)

        print_msg("Retained %d installed software packages" % len(software),
                  silent=silent)

    return generate_doc('list_software_%s' % output_format,
                        [software, detailed])
示例#30
0
def list_software(output_format=FORMAT_TXT, detailed=False, only_installed=False):
    """
    Show list of supported software

    :param output_format: output format to use
    :param detailed: whether or not to return detailed information (incl. version, versionsuffix, toolchain info)
    :param only_installed: only retain software for which a corresponding module is available
    :return: multi-line string presenting requested info
    """
    silent = build_option('silent')

    ec_paths = find_matching_easyconfigs('*', '*', build_option('robot_path') or [])
    ecs = []
    cnt = len(ec_paths)
    for idx, ec_path in enumerate(ec_paths):
        # full EasyConfig instance is only required when module name is needed
        # this is significantly slower (5-10x) than a 'shallow' parse via EasyConfigParser
        if only_installed:
            ec = process_easyconfig(ec_path, validate=False, parse_only=True)[0]['ec']
        else:
            ec = EasyConfigParser(filename=ec_path).get_config_dict()

        ecs.append(ec)
        print_msg('\r', prefix=False, newline=False, silent=silent)
        print_msg("Processed %d/%d easyconfigs..." % (idx+1, cnt), newline=False, silent=silent)
    print_msg('', prefix=False, silent=silent)

    software = {}
    for ec in ecs:
        software.setdefault(ec['name'], [])
        if ec['toolchain']['name'] == DUMMY_TOOLCHAIN_NAME:
            toolchain = DUMMY_TOOLCHAIN_NAME
        else:
            toolchain = '%s/%s' % (ec['toolchain']['name'], ec['toolchain']['version'])

        keys = ['description', 'homepage', 'version', 'versionsuffix']

        info = {'toolchain': toolchain}
        for key in keys:
            info[key] = ec.get(key, '')

        # make sure values like homepage & versionsuffix get properly templated
        if isinstance(ec, dict):
            template_values = template_constant_dict(ec, skip_lower=False)
            for key in keys:
                if '%(' in info[key]:
                    try:
                        info[key] = info[key] % template_values
                    except (KeyError, TypeError, ValueError) as err:
                        _log.debug("Ignoring failure to resolve templates: %s", err)

        software[ec['name']].append(info)

        if only_installed:
            software[ec['name']][-1].update({'mod_name': ec.full_mod_name})

    print_msg("Found %d different software packages" % len(software), silent=silent)

    if only_installed:
        avail_mod_names = modules_tool().available()

        # rebuild software, only retain entries with a corresponding available module
        software, all_software = {}, software
        for key in all_software:
            for entry in all_software[key]:
                if entry['mod_name'] in avail_mod_names:
                    software.setdefault(key, []).append(entry)

        print_msg("Retained %d installed software packages" % len(software), silent=silent)

    return generate_doc('list_software_%s' % output_format, [software, detailed])
示例#31
0
def template_easyconfig_test(self, spec):
    """Tests for an individual easyconfig: parsing, instantiating easyblock, check patches, ..."""

    # set to False, so it's False in case of this test failing
    global single_tests_ok
    prev_single_tests_ok = single_tests_ok
    single_tests_ok = False

    # parse easyconfig
    ecs = process_easyconfig(spec)
    if len(ecs) == 1:
        ec = ecs[0]['ec']
    else:
        self.assertTrue(
            False,
            "easyconfig %s does not contain blocks, yields only one parsed easyconfig"
            % spec)

    # check easyconfig file name
    expected_fn = '%s-%s.eb' % (ec['name'], det_full_ec_version(ec))
    msg = "Filename '%s' of parsed easyconfig matches expected filename '%s'" % (
        spec, expected_fn)
    self.assertEqual(os.path.basename(spec), expected_fn, msg)

    name, easyblock = fetch_parameters_from_easyconfig(ec.rawtxt,
                                                       ['name', 'easyblock'])

    # make sure easyconfig file is in expected location
    expected_subdir = os.path.join('easybuild', 'easyconfigs',
                                   name.lower()[0], name)
    subdir = os.path.join(*spec.split(os.path.sep)[-5:-1])
    fail_msg = "Easyconfig file %s not in expected subdirectory %s" % (
        spec, expected_subdir)
    self.assertEqual(expected_subdir, subdir, fail_msg)

    # sanity check for software name, moduleclass
    self.assertEqual(ec['name'], name)
    self.assertTrue(ec['moduleclass'] in build_option('valid_module_classes'))

    # instantiate easyblock with easyconfig file
    app_class = get_easyblock_class(easyblock, name=name)

    # check that automagic fallback to ConfigureMake isn't done (deprecated behaviour)
    fn = os.path.basename(spec)
    error_msg = "%s relies on automagic fallback to ConfigureMake, should use easyblock = 'ConfigureMake' instead" % fn
    self.assertTrue(easyblock or not app_class is ConfigureMake, error_msg)

    app = app_class(ec)

    # more sanity checks
    self.assertTrue(name, app.name)
    self.assertTrue(ec['version'], app.version)

    # make sure all patch files are available
    specdir = os.path.dirname(spec)
    specfn = os.path.basename(spec)
    for patch in ec['patches']:
        if isinstance(patch, (tuple, list)):
            patch = patch[0]
        # only check actual patch files, not other files being copied via the patch functionality
        if patch.endswith('.patch'):
            patch_full = os.path.join(specdir, patch)
            msg = "Patch file %s is available for %s" % (patch_full, specfn)
            self.assertTrue(os.path.isfile(patch_full), msg)
    ext_patches = []
    for ext in ec['exts_list']:
        if isinstance(ext, (tuple, list)) and len(ext) == 3:
            self.assertTrue(isinstance(ext[2], dict),
                            "3rd element of extension spec is a dictionary")
            for ext_patch in ext[2].get('patches', []):
                if isinstance(ext_patch, (tuple, list)):
                    ext_patch = ext_patch[0]
                # only check actual patch files, not other files being copied via the patch functionality
                if ext_patch.endswith('.patch'):
                    ext_patch_full = os.path.join(specdir, ext_patch)
                    msg = "Patch file %s is available for %s" % (
                        ext_patch_full, specfn)
                    self.assertTrue(os.path.isfile(ext_patch_full), msg)

    # check whether all extra_options defined for used easyblock are defined
    extra_opts = app.extra_options()
    for key in extra_opts:
        self.assertTrue(key in app.cfg)

    app.close_log()
    os.remove(app.logfile)

    # dump the easyconfig file
    handle, test_ecfile = tempfile.mkstemp()
    os.close(handle)

    ec.dump(test_ecfile)
    dumped_ec = EasyConfigParser(test_ecfile).get_config_dict()
    os.remove(test_ecfile)

    # inject dummy values for templates that are only known at a later stage
    dummy_template_values = {
        'builddir': '/dummy/builddir',
        'installdir': '/dummy/installdir',
    }
    ec.template_values.update(dummy_template_values)

    ec_dict = ec.parser.get_config_dict()
    keys = []
    for key in ec_dict:
        # skip parameters for which value is equal to default value
        orig_val = ec_dict[key]
        if key in DEFAULT_CONFIG and orig_val == DEFAULT_CONFIG[key][0]:
            continue
        if key in extra_opts and orig_val == extra_opts[key][0]:
            continue
        if key not in DEFAULT_CONFIG and key not in extra_opts:
            continue

        keys.append(key)
        orig_val = resolve_template(ec_dict[key], ec.template_values)
        dumped_val = resolve_template(dumped_ec[key], ec.template_values)

        self.assertEqual(orig_val, dumped_val)

    # cache the parsed easyconfig, to avoid that it is parsed again
    self.parsed_easyconfigs.append(ecs[0])

    # test passed, so set back to True
    single_tests_ok = True and prev_single_tests_ok
示例#32
0
def list_software(output_format=FORMAT_TXT, detailed=False, only_installed=False):
    """
    Show list of supported software

    :param output_format: output format to use
    :param detailed: whether or not to return detailed information (incl. version, versionsuffix, toolchain info)
    :param only_installed: only retain software for which a corresponding module is available
    :return: multi-line string presenting requested info
    """
    silent = build_option("silent")

    ec_paths = find_matching_easyconfigs("*", "*", build_option("robot_path") or [])
    ecs = []
    cnt = len(ec_paths)
    for idx, ec_path in enumerate(ec_paths):
        # full EasyConfig instance is only required when module name is needed
        # this is significantly slower (5-10x) than a 'shallow' parse via EasyConfigParser
        if only_installed:
            ec = process_easyconfig(ec_path, validate=False, parse_only=True)[0]["ec"]
        else:
            ec = EasyConfigParser(filename=ec_path).get_config_dict()

        ecs.append(ec)
        print_msg("\r", prefix=False, newline=False, silent=silent)
        print_msg("Processed %d/%d easyconfigs..." % (idx + 1, cnt), newline=False, silent=silent)
    print_msg("", prefix=False, silent=silent)

    software = {}
    for ec in ecs:
        software.setdefault(ec["name"], [])
        if ec["toolchain"]["name"] == DUMMY_TOOLCHAIN_NAME:
            toolchain = DUMMY_TOOLCHAIN_NAME
        else:
            toolchain = "%s/%s" % (ec["toolchain"]["name"], ec["toolchain"]["version"])

        versionsuffix = ec.get("versionsuffix", "")

        # make sure versionsuffix gets properly templated
        if versionsuffix and isinstance(ec, dict):
            template_values = template_constant_dict(ec)
            versionsuffix = versionsuffix % template_values

        software[ec["name"]].append(
            {
                "description": ec["description"],
                "homepage": ec["homepage"],
                "toolchain": toolchain,
                "version": ec["version"],
                "versionsuffix": versionsuffix,
            }
        )

        if only_installed:
            software[ec["name"]][-1].update({"mod_name": ec.full_mod_name})

    print_msg("Found %d different software packages" % len(software), silent=silent)

    if only_installed:
        avail_mod_names = modules_tool().available()

        # rebuild software, only retain entries with a corresponding available module
        software, all_software = {}, software
        for key in all_software:
            for entry in all_software[key]:
                if entry["mod_name"] in avail_mod_names:
                    software.setdefault(key, []).append(entry)

        print_msg("Retained %d installed software packages" % len(software), silent=silent)

    return generate_doc("list_software_%s" % output_format, [software, detailed])