def __init__(self, mself, ext, extra_params=None): """ Constructor for Extension class :param mself: parent Easyblock instance :param ext: dictionary with extension metadata (name, version, src, patches, options, ...) :param extra_params: extra custom easyconfig parameters to take into account for this extension """ self.master = mself self.log = self.master.log self.cfg = self.master.cfg.copy(validate=False) self.ext = copy.deepcopy(ext) self.dry_run = self.master.dry_run if 'name' not in self.ext: raise EasyBuildError( "'name' is missing in supplied class instance 'ext'.") name, version = self.ext['name'], self.ext.get('version', None) # parent sanity check paths/commands are not relevant for extension self.cfg['sanity_check_commands'] = [] self.cfg['sanity_check_paths'] = [] # construct dict with template values that can be used self.cfg.template_values.update( template_constant_dict({ 'name': name, 'version': version })) # list of source/patch files: we use an empty list as default value like in EasyBlock self.src = resolve_template(self.ext.get('src', []), self.cfg.template_values) self.patches = resolve_template(self.ext.get('patches', []), self.cfg.template_values) self.options = resolve_template( copy.deepcopy(self.ext.get('options', {})), self.cfg.template_values) if extra_params: self.cfg.extend_params(extra_params, overwrite=False) # custom easyconfig parameters for extension are included in self.options # make sure they are merged into self.cfg so they can be queried; # unknown easyconfig parameters are ignored since self.options may include keys only there for extensions; # this allows to specify custom easyconfig parameters on a per-extension basis for key in self.options: if key in self.cfg: self.cfg[key] = resolve_template(self.options[key], self.cfg.template_values) self.log.debug( "Customising known easyconfig parameter '%s' for extension %s/%s: %s", key, name, version, self.cfg[key]) else: self.log.debug( "Skipping unknown custom easyconfig parameter '%s' for extension %s/%s: %s", key, name, version, self.options[key]) self.sanity_check_fail_msgs = []
def __init__(self, mself, ext, extra_params=None): """ Constructor for Extension class :param mself: parent Easyblock instance :param ext: dictionary with extension metadata (name, version, src, patches, options, ...) :param extra_params: extra custom easyconfig parameters to take into account for this extension """ self.master = mself self.log = self.master.log self.cfg = self.master.cfg.copy(validate=False) self.ext = copy.deepcopy(ext) self.dry_run = self.master.dry_run if 'name' not in self.ext: raise EasyBuildError("'name' is missing in supplied class instance 'ext'.") name, version = self.ext['name'], self.ext.get('version', None) # parent sanity check paths/commands and postinstallcmds are not relevant for extension self.cfg['sanity_check_commands'] = [] self.cfg['sanity_check_paths'] = [] self.cfg['postinstallcmds'] = [] # construct dict with template values that can be used self.cfg.template_values.update(template_constant_dict({'name': name, 'version': version})) # Add install/builddir templates with values from master. for key in TEMPLATE_NAMES_EASYBLOCK_RUN_STEP: self.cfg.template_values[key[0]] = str(getattr(self.master, key[0], None)) # list of source/patch files: we use an empty list as default value like in EasyBlock self.src = resolve_template(self.ext.get('src', []), self.cfg.template_values) self.src_extract_cmd = self.ext.get('extract_cmd', None) self.patches = resolve_template(self.ext.get('patches', []), self.cfg.template_values) self.options = resolve_template(copy.deepcopy(self.ext.get('options', {})), self.cfg.template_values) if extra_params: self.cfg.extend_params(extra_params, overwrite=False) # custom easyconfig parameters for extension are included in self.options # make sure they are merged into self.cfg so they can be queried; # unknown easyconfig parameters are ignored since self.options may include keys only there for extensions; # this allows to specify custom easyconfig parameters on a per-extension basis for key, value in self.options.items(): if key in self.cfg: self.cfg[key] = value self.log.debug("Customising known easyconfig parameter '%s' for extension %s/%s: %s", key, name, version, value) else: self.log.debug("Skipping unknown custom easyconfig parameter '%s' for extension %s/%s: %s", key, name, version, value) self.sanity_check_fail_msgs = [] self.async_cmd_info = None self.async_cmd_output = None self.async_cmd_check_cnt = None # initial read size should be relatively small, # to avoid hanging for a long time until desired output is available in async_cmd_check self.async_cmd_read_size = 1024
def template_easyconfig_test(self, spec): """Tests for an individual easyconfig: parsing, instantiating easyblock, check patches, ...""" # set to False, so it's False in case of this test failing global single_tests_ok prev_single_tests_ok = single_tests_ok single_tests_ok = False # parse easyconfig ecs = process_easyconfig(spec) if len(ecs) == 1: ec = ecs[0]['ec'] # cache the parsed easyconfig, to avoid that it is parsed again self.parsed_easyconfigs.append(ecs[0]) else: self.assertTrue( False, "easyconfig %s does not contain blocks, yields only one parsed easyconfig" % spec) # check easyconfig file name expected_fn = '%s-%s.eb' % (ec['name'], det_full_ec_version(ec)) msg = "Filename '%s' of parsed easyconfig matches expected filename '%s'" % ( spec, expected_fn) self.assertEqual(os.path.basename(spec), expected_fn, msg) name, easyblock = fetch_parameters_from_easyconfig(ec.rawtxt, ['name', 'easyblock']) # make sure easyconfig file is in expected location expected_subdir = os.path.join('easybuild', 'easyconfigs', letter_dir_for(name), name) subdir = os.path.join(*spec.split(os.path.sep)[-5:-1]) fail_msg = "Easyconfig file %s not in expected subdirectory %s" % ( spec, expected_subdir) self.assertEqual(expected_subdir, subdir, fail_msg) # sanity check for software name, moduleclass self.assertEqual(ec['name'], name) self.assertTrue(ec['moduleclass'] in build_option('valid_module_classes')) # instantiate easyblock with easyconfig file app_class = get_easyblock_class(easyblock, name=name) # check that automagic fallback to ConfigureMake isn't done (deprecated behaviour) fn = os.path.basename(spec) error_msg = "%s relies on automagic fallback to ConfigureMake, should use easyblock = 'ConfigureMake' instead" % fn self.assertTrue(easyblock or app_class is not ConfigureMake, error_msg) app = app_class(ec) # more sanity checks self.assertTrue(name, app.name) self.assertTrue(ec['version'], app.version) # make sure that $root is not used, since it is not compatible with module files in Lua syntax res = re.findall('.*\$root.*', ec.rawtxt, re.M) error_msg = "Found use of '$root', not compatible with modules in Lua syntax, use '%%(installdir)s' instead: %s" self.assertFalse(res, error_msg % res) # make sure old GitHub urls for EasyBuild that include 'hpcugent' are no longer used old_urls = [ 'github.com/hpcugent/easybuild', 'hpcugent.github.com/easybuild', 'hpcugent.github.io/easybuild', ] for old_url in old_urls: self.assertFalse(old_url in ec.rawtxt, "Old URL '%s' not found in %s" % (old_url, spec)) # make sure binutils is included as a build dep if toolchain is GCCcore if ec['toolchain']['name'] == 'GCCcore': # with 'Tarball' easyblock: only unpacking, no building; Eigen is also just a tarball requires_binutils = ec['easyblock'] not in [ 'Tarball' ] and ec['name'] not in ['Eigen'] # let's also exclude the very special case where the system GCC is used as GCCcore, and only apply this # exception to the dependencies of binutils (since we should eventually build a new binutils with GCCcore) if ec['toolchain']['version'] == 'system': binutils_complete_dependencies = [ 'M4', 'Bison', 'flex', 'help2man', 'zlib', 'binutils' ] requires_binutils &= bool( ec['name'] not in binutils_complete_dependencies) # if no sources/extensions/components are specified, it's just a bundle (nothing is being compiled) requires_binutils &= bool(ec['sources'] or ec['exts_list'] or ec.get('components')) if requires_binutils: dep_names = [d['name'] for d in ec.builddependencies()] self.assertTrue( 'binutils' in dep_names, "binutils is a build dep in %s: %s" % (spec, dep_names)) # make sure all patch files are available specdir = os.path.dirname(spec) specfn = os.path.basename(spec) for patch in ec['patches']: if isinstance(patch, (tuple, list)): patch = patch[0] # only check actual patch files, not other files being copied via the patch functionality if patch.endswith('.patch'): patch_full = os.path.join(specdir, patch) msg = "Patch file %s is available for %s" % (patch_full, specfn) self.assertTrue(os.path.isfile(patch_full), msg) for ext in ec['exts_list']: if isinstance(ext, (tuple, list)) and len(ext) == 3: self.assertTrue(isinstance(ext[2], dict), "3rd element of extension spec is a dictionary") for ext_patch in ext[2].get('patches', []): if isinstance(ext_patch, (tuple, list)): ext_patch = ext_patch[0] # only check actual patch files, not other files being copied via the patch functionality if ext_patch.endswith('.patch'): ext_patch_full = os.path.join(specdir, ext_patch) msg = "Patch file %s is available for %s" % ( ext_patch_full, specfn) self.assertTrue(os.path.isfile(ext_patch_full), msg) # check whether all extra_options defined for used easyblock are defined extra_opts = app.extra_options() for key in extra_opts: self.assertTrue(key in app.cfg) app.close_log() os.remove(app.logfile) # dump the easyconfig file handle, test_ecfile = tempfile.mkstemp() os.close(handle) ec.dump(test_ecfile) dumped_ec = EasyConfigParser(test_ecfile).get_config_dict() os.remove(test_ecfile) # inject dummy values for templates that are only known at a later stage dummy_template_values = { 'builddir': '/dummy/builddir', 'installdir': '/dummy/installdir', } ec.template_values.update(dummy_template_values) ec_dict = ec.parser.get_config_dict() orig_toolchain = ec_dict['toolchain'] for key in ec_dict: # skip parameters for which value is equal to default value orig_val = ec_dict[key] if key in DEFAULT_CONFIG and orig_val == DEFAULT_CONFIG[key][0]: continue if key in extra_opts and orig_val == extra_opts[key][0]: continue if key not in DEFAULT_CONFIG and key not in extra_opts: continue orig_val = resolve_template(ec_dict[key], ec.template_values) dumped_val = resolve_template(dumped_ec[key], ec.template_values) # take into account that dumped value for *dependencies may include hard-coded subtoolchains # if no easyconfig was found for the dependency with the 'parent' toolchain, # if may get resolved using a subtoolchain, which is then hardcoded in the dumped easyconfig if key in DEPENDENCY_PARAMETERS: # number of dependencies should remain the same self.assertEqual(len(orig_val), len(dumped_val)) for orig_dep, dumped_dep in zip(orig_val, dumped_val): # name/version should always match self.assertEqual(orig_dep[:2], dumped_dep[:2]) # 3rd value is versionsuffix; if len(dumped_dep) >= 3: # if no versionsuffix was specified in original dep spec, then dumped value should be empty string if len(orig_dep) >= 3: self.assertEqual(dumped_dep[2], orig_dep[2]) else: self.assertEqual(dumped_dep[2], '') # 4th value is toolchain spec if len(dumped_dep) >= 4: if len(orig_dep) >= 4: self.assertEqual(dumped_dep[3], orig_dep[3]) else: # if a subtoolchain is specifed (only) in the dumped easyconfig, # it should *not* be the same as the parent toolchain self.assertNotEqual(dumped_dep[3], (orig_toolchain['name'], orig_toolchain['version'])) else: self.assertEqual(orig_val, dumped_val) # test passed, so set back to True single_tests_ok = True and prev_single_tests_ok
def template_easyconfig_test(self, spec): """Tests for an individual easyconfig: parsing, instantiating easyblock, check patches, ...""" # set to False, so it's False in case of this test failing global single_tests_ok prev_single_tests_ok = single_tests_ok single_tests_ok = False # parse easyconfig ecs = process_easyconfig(spec) if len(ecs) == 1: ec = ecs[0]['ec'] else: self.assertTrue(False, "easyconfig %s does not contain blocks, yields only one parsed easyconfig" % spec) # check easyconfig file name expected_fn = '%s-%s.eb' % (ec['name'], det_full_ec_version(ec)) msg = "Filename '%s' of parsed easyconfig matches expected filename '%s'" % (spec, expected_fn) self.assertEqual(os.path.basename(spec), expected_fn, msg) name, easyblock = fetch_parameters_from_easyconfig(ec.rawtxt, ['name', 'easyblock']) # make sure easyconfig file is in expected location expected_subdir = os.path.join('easybuild', 'easyconfigs', letter_dir_for(name), name) subdir = os.path.join(*spec.split(os.path.sep)[-5:-1]) fail_msg = "Easyconfig file %s not in expected subdirectory %s" % (spec, expected_subdir) self.assertEqual(expected_subdir, subdir, fail_msg) # sanity check for software name, moduleclass self.assertEqual(ec['name'], name) self.assertTrue(ec['moduleclass'] in build_option('valid_module_classes')) # instantiate easyblock with easyconfig file app_class = get_easyblock_class(easyblock, name=name) # check that automagic fallback to ConfigureMake isn't done (deprecated behaviour) fn = os.path.basename(spec) error_msg = "%s relies on automagic fallback to ConfigureMake, should use easyblock = 'ConfigureMake' instead" % fn self.assertTrue(easyblock or not app_class is ConfigureMake, error_msg) app = app_class(ec) # more sanity checks self.assertTrue(name, app.name) self.assertTrue(ec['version'], app.version) # make sure all patch files are available specdir = os.path.dirname(spec) specfn = os.path.basename(spec) for patch in ec['patches']: if isinstance(patch, (tuple, list)): patch = patch[0] # only check actual patch files, not other files being copied via the patch functionality if patch.endswith('.patch'): patch_full = os.path.join(specdir, patch) msg = "Patch file %s is available for %s" % (patch_full, specfn) self.assertTrue(os.path.isfile(patch_full), msg) ext_patches = [] for ext in ec['exts_list']: if isinstance(ext, (tuple, list)) and len(ext) == 3: self.assertTrue(isinstance(ext[2], dict), "3rd element of extension spec is a dictionary") for ext_patch in ext[2].get('patches', []): if isinstance(ext_patch, (tuple, list)): ext_patch = ext_patch[0] # only check actual patch files, not other files being copied via the patch functionality if ext_patch.endswith('.patch'): ext_patch_full = os.path.join(specdir, ext_patch) msg = "Patch file %s is available for %s" % (ext_patch_full, specfn) self.assertTrue(os.path.isfile(ext_patch_full), msg) # check whether all extra_options defined for used easyblock are defined extra_opts = app.extra_options() for key in extra_opts: self.assertTrue(key in app.cfg) app.close_log() os.remove(app.logfile) # dump the easyconfig file handle, test_ecfile = tempfile.mkstemp() os.close(handle) ec.dump(test_ecfile) dumped_ec = EasyConfigParser(test_ecfile).get_config_dict() os.remove(test_ecfile) # inject dummy values for templates that are only known at a later stage dummy_template_values = { 'builddir': '/dummy/builddir', 'installdir': '/dummy/installdir', } ec.template_values.update(dummy_template_values) ec_dict = ec.parser.get_config_dict() orig_toolchain = ec_dict['toolchain'] for key in ec_dict: # skip parameters for which value is equal to default value orig_val = ec_dict[key] if key in DEFAULT_CONFIG and orig_val == DEFAULT_CONFIG[key][0]: continue if key in extra_opts and orig_val == extra_opts[key][0]: continue if key not in DEFAULT_CONFIG and key not in extra_opts: continue orig_val = resolve_template(ec_dict[key], ec.template_values) dumped_val = resolve_template(dumped_ec[key], ec.template_values) # take into account that dumped value for *dependencies may include hard-coded subtoolchains # if no easyconfig was found for the dependency with the 'parent' toolchain, # if may get resolved using a subtoolchain, which is then hardcoded in the dumped easyconfig if key in DEPENDENCY_PARAMETERS: # number of dependencies should remain the same self.assertEqual(len(orig_val), len(dumped_val)) for orig_dep, dumped_dep in zip(orig_val, dumped_val): # name/version should always match self.assertEqual(orig_dep[:2], dumped_dep[:2]) # 3rd value is versionsuffix; if len(dumped_dep) >= 3: # if no versionsuffix was specified in original dep spec, then dumped value should be empty string if len(orig_dep) >= 3: self.assertEqual(dumped_dep[2], orig_dep[2]) else: self.assertEqual(dumped_dep[2], '') # 4th value is toolchain spec if len(dumped_dep) >= 4: if len(orig_dep) >= 4: self.assertEqual(dumped_dep[3], orig_dep[3]) else: # if a subtoolchain is specifed (only) in the dumped easyconfig, # it should *not* be the same as the parent toolchain self.assertNotEqual(dumped_dep[3], (orig_toolchain['name'], orig_toolchain['version'])) else: self.assertEqual(orig_val, dumped_val) # cache the parsed easyconfig, to avoid that it is parsed again self.parsed_easyconfigs.append(ecs[0]) # test passed, so set back to True single_tests_ok = True and prev_single_tests_ok
def template_easyconfig_test(self, spec): """Tests for an individual easyconfig: parsing, instantiating easyblock, check patches, ...""" # set to False, so it's False in case of this test failing global single_tests_ok prev_single_tests_ok = single_tests_ok single_tests_ok = False # parse easyconfig ecs = process_easyconfig(spec) if len(ecs) == 1: ec = ecs[0]['ec'] else: self.assertTrue(False, "easyconfig %s does not contain blocks, yields only one parsed easyconfig" % spec) # check easyconfig file name expected_fn = '%s-%s.eb' % (ec['name'], det_full_ec_version(ec)) msg = "Filename '%s' of parsed easyconfig matches expected filename '%s'" % (spec, expected_fn) self.assertEqual(os.path.basename(spec), expected_fn, msg) name, easyblock = fetch_parameters_from_easyconfig(ec.rawtxt, ['name', 'easyblock']) # make sure easyconfig file is in expected location expected_subdir = os.path.join('easybuild', 'easyconfigs', letter_dir_for(name), name) subdir = os.path.join(*spec.split(os.path.sep)[-5:-1]) fail_msg = "Easyconfig file %s not in expected subdirectory %s" % (spec, expected_subdir) self.assertEqual(expected_subdir, subdir, fail_msg) # sanity check for software name, moduleclass self.assertEqual(ec['name'], name) self.assertTrue(ec['moduleclass'] in build_option('valid_module_classes')) # instantiate easyblock with easyconfig file app_class = get_easyblock_class(easyblock, name=name) # check that automagic fallback to ConfigureMake isn't done (deprecated behaviour) fn = os.path.basename(spec) error_msg = "%s relies on automagic fallback to ConfigureMake, should use easyblock = 'ConfigureMake' instead" % fn self.assertTrue(easyblock or not app_class is ConfigureMake, error_msg) app = app_class(ec) # more sanity checks self.assertTrue(name, app.name) self.assertTrue(ec['version'], app.version) # make sure that $root is not used, since it is not compatible with module files in Lua syntax res = re.findall('.*\$root.*', ec.rawtxt, re.M) error_msg = "Found use of '$root', not compatible with modules in Lua syntax, use '%%(installdir)s' instead: %s" self.assertFalse(res, error_msg % res) # make sure old GitHub urls for EasyBuild that include 'hpcugent' are no longer used old_urls = [ 'github.com/hpcugent/easybuild', 'hpcugent.github.com/easybuild', 'hpcugent.github.io/easybuild', ] for old_url in old_urls: self.assertFalse(old_url in ec.rawtxt, "Old URL '%s' not found in %s" % (old_url, spec)) # make sure binutils is included as a build dep if toolchain is GCCcore if ec['toolchain']['name'] == 'GCCcore': # with 'Tarball' easyblock: only unpacking, no building; Eigen is also just a tarball requires_binutils = ec['easyblock'] not in ['Tarball'] and ec['name'] not in ['Eigen'] # let's also exclude the very special case where the system GCC is used as GCCcore, and only apply this # exception to the dependencies of binutils (since we should eventually build a new binutils with GCCcore) if ec['toolchain']['version'] == 'system': binutils_complete_dependencies = ['M4', 'Bison', 'flex', 'help2man', 'zlib', 'binutils'] requires_binutils &= bool(ec['name'] not in binutils_complete_dependencies) # if no sources/extensions/components are specified, it's just a bundle (nothing is being compiled) requires_binutils &= bool(ec['sources'] or ec['exts_list'] or ec.get('components')) if requires_binutils: dep_names = [d['name'] for d in ec['builddependencies']] self.assertTrue('binutils' in dep_names, "binutils is a build dep in %s: %s" % (spec, dep_names)) # make sure all patch files are available specdir = os.path.dirname(spec) specfn = os.path.basename(spec) for patch in ec['patches']: if isinstance(patch, (tuple, list)): patch = patch[0] # only check actual patch files, not other files being copied via the patch functionality if patch.endswith('.patch'): patch_full = os.path.join(specdir, patch) msg = "Patch file %s is available for %s" % (patch_full, specfn) self.assertTrue(os.path.isfile(patch_full), msg) ext_patches = [] for ext in ec['exts_list']: if isinstance(ext, (tuple, list)) and len(ext) == 3: self.assertTrue(isinstance(ext[2], dict), "3rd element of extension spec is a dictionary") for ext_patch in ext[2].get('patches', []): if isinstance(ext_patch, (tuple, list)): ext_patch = ext_patch[0] # only check actual patch files, not other files being copied via the patch functionality if ext_patch.endswith('.patch'): ext_patch_full = os.path.join(specdir, ext_patch) msg = "Patch file %s is available for %s" % (ext_patch_full, specfn) self.assertTrue(os.path.isfile(ext_patch_full), msg) # check whether all extra_options defined for used easyblock are defined extra_opts = app.extra_options() for key in extra_opts: self.assertTrue(key in app.cfg) app.close_log() os.remove(app.logfile) # dump the easyconfig file handle, test_ecfile = tempfile.mkstemp() os.close(handle) ec.dump(test_ecfile) dumped_ec = EasyConfigParser(test_ecfile).get_config_dict() os.remove(test_ecfile) # inject dummy values for templates that are only known at a later stage dummy_template_values = { 'builddir': '/dummy/builddir', 'installdir': '/dummy/installdir', } ec.template_values.update(dummy_template_values) ec_dict = ec.parser.get_config_dict() orig_toolchain = ec_dict['toolchain'] for key in ec_dict: # skip parameters for which value is equal to default value orig_val = ec_dict[key] if key in DEFAULT_CONFIG and orig_val == DEFAULT_CONFIG[key][0]: continue if key in extra_opts and orig_val == extra_opts[key][0]: continue if key not in DEFAULT_CONFIG and key not in extra_opts: continue orig_val = resolve_template(ec_dict[key], ec.template_values) dumped_val = resolve_template(dumped_ec[key], ec.template_values) # take into account that dumped value for *dependencies may include hard-coded subtoolchains # if no easyconfig was found for the dependency with the 'parent' toolchain, # if may get resolved using a subtoolchain, which is then hardcoded in the dumped easyconfig if key in DEPENDENCY_PARAMETERS: # number of dependencies should remain the same self.assertEqual(len(orig_val), len(dumped_val)) for orig_dep, dumped_dep in zip(orig_val, dumped_val): # name/version should always match self.assertEqual(orig_dep[:2], dumped_dep[:2]) # 3rd value is versionsuffix; if len(dumped_dep) >= 3: # if no versionsuffix was specified in original dep spec, then dumped value should be empty string if len(orig_dep) >= 3: self.assertEqual(dumped_dep[2], orig_dep[2]) else: self.assertEqual(dumped_dep[2], '') # 4th value is toolchain spec if len(dumped_dep) >= 4: if len(orig_dep) >= 4: self.assertEqual(dumped_dep[3], orig_dep[3]) else: # if a subtoolchain is specifed (only) in the dumped easyconfig, # it should *not* be the same as the parent toolchain self.assertNotEqual(dumped_dep[3], (orig_toolchain['name'], orig_toolchain['version'])) else: self.assertEqual(orig_val, dumped_val) # cache the parsed easyconfig, to avoid that it is parsed again self.parsed_easyconfigs.append(ecs[0]) # test passed, so set back to True single_tests_ok = True and prev_single_tests_ok
def template_easyconfig_test(self, spec): """Tests for an individual easyconfig: parsing, instantiating easyblock, check patches, ...""" # set to False, so it's False in case of this test failing global single_tests_ok prev_single_tests_ok = single_tests_ok single_tests_ok = False # parse easyconfig ecs = process_easyconfig(spec) if len(ecs) == 1: ec = ecs[0]['ec'] else: self.assertTrue( False, "easyconfig %s does not contain blocks, yields only one parsed easyconfig" % spec) # check easyconfig file name expected_fn = '%s-%s.eb' % (ec['name'], det_full_ec_version(ec)) msg = "Filename '%s' of parsed easyconfig matches expected filename '%s'" % ( spec, expected_fn) self.assertEqual(os.path.basename(spec), expected_fn, msg) name, easyblock = fetch_parameters_from_easyconfig(ec.rawtxt, ['name', 'easyblock']) # make sure easyconfig file is in expected location expected_subdir = os.path.join('easybuild', 'easyconfigs', name.lower()[0], name) subdir = os.path.join(*spec.split(os.path.sep)[-5:-1]) fail_msg = "Easyconfig file %s not in expected subdirectory %s" % ( spec, expected_subdir) self.assertEqual(expected_subdir, subdir, fail_msg) # sanity check for software name, moduleclass self.assertEqual(ec['name'], name) self.assertTrue(ec['moduleclass'] in build_option('valid_module_classes')) # instantiate easyblock with easyconfig file app_class = get_easyblock_class(easyblock, name=name) # check that automagic fallback to ConfigureMake isn't done (deprecated behaviour) fn = os.path.basename(spec) error_msg = "%s relies on automagic fallback to ConfigureMake, should use easyblock = 'ConfigureMake' instead" % fn self.assertTrue(easyblock or not app_class is ConfigureMake, error_msg) app = app_class(ec) # more sanity checks self.assertTrue(name, app.name) self.assertTrue(ec['version'], app.version) # make sure all patch files are available specdir = os.path.dirname(spec) specfn = os.path.basename(spec) for patch in ec['patches']: if isinstance(patch, (tuple, list)): patch = patch[0] # only check actual patch files, not other files being copied via the patch functionality if patch.endswith('.patch'): patch_full = os.path.join(specdir, patch) msg = "Patch file %s is available for %s" % (patch_full, specfn) self.assertTrue(os.path.isfile(patch_full), msg) ext_patches = [] for ext in ec['exts_list']: if isinstance(ext, (tuple, list)) and len(ext) == 3: self.assertTrue(isinstance(ext[2], dict), "3rd element of extension spec is a dictionary") for ext_patch in ext[2].get('patches', []): if isinstance(ext_patch, (tuple, list)): ext_patch = ext_patch[0] # only check actual patch files, not other files being copied via the patch functionality if ext_patch.endswith('.patch'): ext_patch_full = os.path.join(specdir, ext_patch) msg = "Patch file %s is available for %s" % ( ext_patch_full, specfn) self.assertTrue(os.path.isfile(ext_patch_full), msg) # check whether all extra_options defined for used easyblock are defined extra_opts = app.extra_options() for key in extra_opts: self.assertTrue(key in app.cfg) app.close_log() os.remove(app.logfile) # dump the easyconfig file handle, test_ecfile = tempfile.mkstemp() os.close(handle) ec.dump(test_ecfile) dumped_ec = EasyConfigParser(test_ecfile).get_config_dict() os.remove(test_ecfile) # inject dummy values for templates that are only known at a later stage dummy_template_values = { 'builddir': '/dummy/builddir', 'installdir': '/dummy/installdir', } ec.template_values.update(dummy_template_values) ec_dict = ec.parser.get_config_dict() keys = [] for key in ec_dict: # skip parameters for which value is equal to default value orig_val = ec_dict[key] if key in DEFAULT_CONFIG and orig_val == DEFAULT_CONFIG[key][0]: continue if key in extra_opts and orig_val == extra_opts[key][0]: continue if key not in DEFAULT_CONFIG and key not in extra_opts: continue keys.append(key) orig_val = resolve_template(ec_dict[key], ec.template_values) dumped_val = resolve_template(dumped_ec[key], ec.template_values) self.assertEqual(orig_val, dumped_val) # cache the parsed easyconfig, to avoid that it is parsed again self.parsed_easyconfigs.append(ecs[0]) # test passed, so set back to True single_tests_ok = True and prev_single_tests_ok