def obtain_repo(self, url, force_download=False): """ Downloads the repo from the given url, then updates submodules :param url: url of git repository """ srcpaths = source_paths() if re.match(r"^(https?)://", url): # URL detected, try and download it reponame = url.split('/')[-1].split('.')[0] # Try and find the repo in the sources directory first foundfile = None failedpaths = [] for path in srcpaths: # create list of candidate filepaths namepath = os.path.join(path, self.name) letterpath = os.path.join(path, letter_dir_for(self.name), self.name) candidate_filepaths = [letterpath, namepath, path] # look for repo in those locations for cfp in candidate_filepaths: fullpath = os.path.join(cfp, reponame) if os.path.isdir(fullpath): self.log.info("Found repo %s at %s", reponame, fullpath) foundfile = os.path.abspath(fullpath) if force_download: print_warning( "Found file %s at %s, but re-downloading it anyway..." % (reponame, foundfile)) foundfile = None break else: failedpaths.append(fullpath) if foundfile: if self.dry_run: self.dry_run_msg(" * %s found at %s", filename, foundfile) return foundfile else: # Figure out where to clone the repo to repopath = os.path.join(srcpaths[0], letter_dir_for(self.name), self.name) self.log.info("Creating path %s to clone repo to" % repopath) mkdir(repopath, parents=True) # Remove repo, if it has previously been downloaded (we must be forcing download) fullpath = os.path.join(repopath, reponame) if os.path.exists(fullpath): rmtree2(fullpath) try: if self.download_repo(reponame, url, repopath): return fullpath except IOError, err: raise EasyBuildError( "Downloading file %s from url %s to %s failed: %s", filename, url, fullpath, err)
def template_easyconfig_test(self, spec): """Tests for an individual easyconfig: parsing, instantiating easyblock, check patches, ...""" # set to False, so it's False in case of this test failing global single_tests_ok prev_single_tests_ok = single_tests_ok single_tests_ok = False # parse easyconfig ecs = process_easyconfig(spec) if len(ecs) == 1: ec = ecs[0]['ec'] # cache the parsed easyconfig, to avoid that it is parsed again self.parsed_easyconfigs.append(ecs[0]) else: self.assertTrue( False, "easyconfig %s does not contain blocks, yields only one parsed easyconfig" % spec) # check easyconfig file name expected_fn = '%s-%s.eb' % (ec['name'], det_full_ec_version(ec)) msg = "Filename '%s' of parsed easyconfig matches expected filename '%s'" % ( spec, expected_fn) self.assertEqual(os.path.basename(spec), expected_fn, msg) name, easyblock = fetch_parameters_from_easyconfig(ec.rawtxt, ['name', 'easyblock']) # make sure easyconfig file is in expected location expected_subdir = os.path.join('easybuild', 'easyconfigs', letter_dir_for(name), name) subdir = os.path.join(*spec.split(os.path.sep)[-5:-1]) fail_msg = "Easyconfig file %s not in expected subdirectory %s" % ( spec, expected_subdir) self.assertEqual(expected_subdir, subdir, fail_msg) # sanity check for software name, moduleclass self.assertEqual(ec['name'], name) self.assertTrue(ec['moduleclass'] in build_option('valid_module_classes')) # instantiate easyblock with easyconfig file app_class = get_easyblock_class(easyblock, name=name) # check that automagic fallback to ConfigureMake isn't done (deprecated behaviour) fn = os.path.basename(spec) error_msg = "%s relies on automagic fallback to ConfigureMake, should use easyblock = 'ConfigureMake' instead" % fn self.assertTrue(easyblock or app_class is not ConfigureMake, error_msg) app = app_class(ec) # more sanity checks self.assertTrue(name, app.name) self.assertTrue(ec['version'], app.version) # make sure that $root is not used, since it is not compatible with module files in Lua syntax res = re.findall('.*\$root.*', ec.rawtxt, re.M) error_msg = "Found use of '$root', not compatible with modules in Lua syntax, use '%%(installdir)s' instead: %s" self.assertFalse(res, error_msg % res) # make sure old GitHub urls for EasyBuild that include 'hpcugent' are no longer used old_urls = [ 'github.com/hpcugent/easybuild', 'hpcugent.github.com/easybuild', 'hpcugent.github.io/easybuild', ] for old_url in old_urls: self.assertFalse(old_url in ec.rawtxt, "Old URL '%s' not found in %s" % (old_url, spec)) # make sure binutils is included as a build dep if toolchain is GCCcore if ec['toolchain']['name'] == 'GCCcore': # with 'Tarball' easyblock: only unpacking, no building; Eigen is also just a tarball requires_binutils = ec['easyblock'] not in [ 'Tarball' ] and ec['name'] not in ['Eigen'] # let's also exclude the very special case where the system GCC is used as GCCcore, and only apply this # exception to the dependencies of binutils (since we should eventually build a new binutils with GCCcore) if ec['toolchain']['version'] == 'system': binutils_complete_dependencies = [ 'M4', 'Bison', 'flex', 'help2man', 'zlib', 'binutils' ] requires_binutils &= bool( ec['name'] not in binutils_complete_dependencies) # if no sources/extensions/components are specified, it's just a bundle (nothing is being compiled) requires_binutils &= bool(ec['sources'] or ec['exts_list'] or ec.get('components')) if requires_binutils: dep_names = [d['name'] for d in ec.builddependencies()] self.assertTrue( 'binutils' in dep_names, "binutils is a build dep in %s: %s" % (spec, dep_names)) # make sure all patch files are available specdir = os.path.dirname(spec) specfn = os.path.basename(spec) for patch in ec['patches']: if isinstance(patch, (tuple, list)): patch = patch[0] # only check actual patch files, not other files being copied via the patch functionality if patch.endswith('.patch'): patch_full = os.path.join(specdir, patch) msg = "Patch file %s is available for %s" % (patch_full, specfn) self.assertTrue(os.path.isfile(patch_full), msg) for ext in ec['exts_list']: if isinstance(ext, (tuple, list)) and len(ext) == 3: self.assertTrue(isinstance(ext[2], dict), "3rd element of extension spec is a dictionary") for ext_patch in ext[2].get('patches', []): if isinstance(ext_patch, (tuple, list)): ext_patch = ext_patch[0] # only check actual patch files, not other files being copied via the patch functionality if ext_patch.endswith('.patch'): ext_patch_full = os.path.join(specdir, ext_patch) msg = "Patch file %s is available for %s" % ( ext_patch_full, specfn) self.assertTrue(os.path.isfile(ext_patch_full), msg) # check whether all extra_options defined for used easyblock are defined extra_opts = app.extra_options() for key in extra_opts: self.assertTrue(key in app.cfg) app.close_log() os.remove(app.logfile) # dump the easyconfig file handle, test_ecfile = tempfile.mkstemp() os.close(handle) ec.dump(test_ecfile) dumped_ec = EasyConfigParser(test_ecfile).get_config_dict() os.remove(test_ecfile) # inject dummy values for templates that are only known at a later stage dummy_template_values = { 'builddir': '/dummy/builddir', 'installdir': '/dummy/installdir', } ec.template_values.update(dummy_template_values) ec_dict = ec.parser.get_config_dict() orig_toolchain = ec_dict['toolchain'] for key in ec_dict: # skip parameters for which value is equal to default value orig_val = ec_dict[key] if key in DEFAULT_CONFIG and orig_val == DEFAULT_CONFIG[key][0]: continue if key in extra_opts and orig_val == extra_opts[key][0]: continue if key not in DEFAULT_CONFIG and key not in extra_opts: continue orig_val = resolve_template(ec_dict[key], ec.template_values) dumped_val = resolve_template(dumped_ec[key], ec.template_values) # take into account that dumped value for *dependencies may include hard-coded subtoolchains # if no easyconfig was found for the dependency with the 'parent' toolchain, # if may get resolved using a subtoolchain, which is then hardcoded in the dumped easyconfig if key in DEPENDENCY_PARAMETERS: # number of dependencies should remain the same self.assertEqual(len(orig_val), len(dumped_val)) for orig_dep, dumped_dep in zip(orig_val, dumped_val): # name/version should always match self.assertEqual(orig_dep[:2], dumped_dep[:2]) # 3rd value is versionsuffix; if len(dumped_dep) >= 3: # if no versionsuffix was specified in original dep spec, then dumped value should be empty string if len(orig_dep) >= 3: self.assertEqual(dumped_dep[2], orig_dep[2]) else: self.assertEqual(dumped_dep[2], '') # 4th value is toolchain spec if len(dumped_dep) >= 4: if len(orig_dep) >= 4: self.assertEqual(dumped_dep[3], orig_dep[3]) else: # if a subtoolchain is specifed (only) in the dumped easyconfig, # it should *not* be the same as the parent toolchain self.assertNotEqual(dumped_dep[3], (orig_toolchain['name'], orig_toolchain['version'])) else: self.assertEqual(orig_val, dumped_val) # test passed, so set back to True single_tests_ok = True and prev_single_tests_ok
def template_easyconfig_test(self, spec): """Tests for an individual easyconfig: parsing, instantiating easyblock, check patches, ...""" # set to False, so it's False in case of this test failing global single_tests_ok prev_single_tests_ok = single_tests_ok single_tests_ok = False # parse easyconfig ecs = process_easyconfig(spec) if len(ecs) == 1: ec = ecs[0]['ec'] else: self.assertTrue(False, "easyconfig %s does not contain blocks, yields only one parsed easyconfig" % spec) # check easyconfig file name expected_fn = '%s-%s.eb' % (ec['name'], det_full_ec_version(ec)) msg = "Filename '%s' of parsed easyconfig matches expected filename '%s'" % (spec, expected_fn) self.assertEqual(os.path.basename(spec), expected_fn, msg) name, easyblock = fetch_parameters_from_easyconfig(ec.rawtxt, ['name', 'easyblock']) # make sure easyconfig file is in expected location expected_subdir = os.path.join('easybuild', 'easyconfigs', letter_dir_for(name), name) subdir = os.path.join(*spec.split(os.path.sep)[-5:-1]) fail_msg = "Easyconfig file %s not in expected subdirectory %s" % (spec, expected_subdir) self.assertEqual(expected_subdir, subdir, fail_msg) # sanity check for software name, moduleclass self.assertEqual(ec['name'], name) self.assertTrue(ec['moduleclass'] in build_option('valid_module_classes')) # instantiate easyblock with easyconfig file app_class = get_easyblock_class(easyblock, name=name) # check that automagic fallback to ConfigureMake isn't done (deprecated behaviour) fn = os.path.basename(spec) error_msg = "%s relies on automagic fallback to ConfigureMake, should use easyblock = 'ConfigureMake' instead" % fn self.assertTrue(easyblock or not app_class is ConfigureMake, error_msg) app = app_class(ec) # more sanity checks self.assertTrue(name, app.name) self.assertTrue(ec['version'], app.version) # make sure all patch files are available specdir = os.path.dirname(spec) specfn = os.path.basename(spec) for patch in ec['patches']: if isinstance(patch, (tuple, list)): patch = patch[0] # only check actual patch files, not other files being copied via the patch functionality if patch.endswith('.patch'): patch_full = os.path.join(specdir, patch) msg = "Patch file %s is available for %s" % (patch_full, specfn) self.assertTrue(os.path.isfile(patch_full), msg) ext_patches = [] for ext in ec['exts_list']: if isinstance(ext, (tuple, list)) and len(ext) == 3: self.assertTrue(isinstance(ext[2], dict), "3rd element of extension spec is a dictionary") for ext_patch in ext[2].get('patches', []): if isinstance(ext_patch, (tuple, list)): ext_patch = ext_patch[0] # only check actual patch files, not other files being copied via the patch functionality if ext_patch.endswith('.patch'): ext_patch_full = os.path.join(specdir, ext_patch) msg = "Patch file %s is available for %s" % (ext_patch_full, specfn) self.assertTrue(os.path.isfile(ext_patch_full), msg) # check whether all extra_options defined for used easyblock are defined extra_opts = app.extra_options() for key in extra_opts: self.assertTrue(key in app.cfg) app.close_log() os.remove(app.logfile) # dump the easyconfig file handle, test_ecfile = tempfile.mkstemp() os.close(handle) ec.dump(test_ecfile) dumped_ec = EasyConfigParser(test_ecfile).get_config_dict() os.remove(test_ecfile) # inject dummy values for templates that are only known at a later stage dummy_template_values = { 'builddir': '/dummy/builddir', 'installdir': '/dummy/installdir', } ec.template_values.update(dummy_template_values) ec_dict = ec.parser.get_config_dict() orig_toolchain = ec_dict['toolchain'] for key in ec_dict: # skip parameters for which value is equal to default value orig_val = ec_dict[key] if key in DEFAULT_CONFIG and orig_val == DEFAULT_CONFIG[key][0]: continue if key in extra_opts and orig_val == extra_opts[key][0]: continue if key not in DEFAULT_CONFIG and key not in extra_opts: continue orig_val = resolve_template(ec_dict[key], ec.template_values) dumped_val = resolve_template(dumped_ec[key], ec.template_values) # take into account that dumped value for *dependencies may include hard-coded subtoolchains # if no easyconfig was found for the dependency with the 'parent' toolchain, # if may get resolved using a subtoolchain, which is then hardcoded in the dumped easyconfig if key in DEPENDENCY_PARAMETERS: # number of dependencies should remain the same self.assertEqual(len(orig_val), len(dumped_val)) for orig_dep, dumped_dep in zip(orig_val, dumped_val): # name/version should always match self.assertEqual(orig_dep[:2], dumped_dep[:2]) # 3rd value is versionsuffix; if len(dumped_dep) >= 3: # if no versionsuffix was specified in original dep spec, then dumped value should be empty string if len(orig_dep) >= 3: self.assertEqual(dumped_dep[2], orig_dep[2]) else: self.assertEqual(dumped_dep[2], '') # 4th value is toolchain spec if len(dumped_dep) >= 4: if len(orig_dep) >= 4: self.assertEqual(dumped_dep[3], orig_dep[3]) else: # if a subtoolchain is specifed (only) in the dumped easyconfig, # it should *not* be the same as the parent toolchain self.assertNotEqual(dumped_dep[3], (orig_toolchain['name'], orig_toolchain['version'])) else: self.assertEqual(orig_val, dumped_val) # cache the parsed easyconfig, to avoid that it is parsed again self.parsed_easyconfigs.append(ecs[0]) # test passed, so set back to True single_tests_ok = True and prev_single_tests_ok
def template_easyconfig_test(self, spec): """Tests for an individual easyconfig: parsing, instantiating easyblock, check patches, ...""" # set to False, so it's False in case of this test failing global single_tests_ok prev_single_tests_ok = single_tests_ok single_tests_ok = False # parse easyconfig ecs = process_easyconfig(spec) if len(ecs) == 1: ec = ecs[0]['ec'] else: self.assertTrue(False, "easyconfig %s does not contain blocks, yields only one parsed easyconfig" % spec) # check easyconfig file name expected_fn = '%s-%s.eb' % (ec['name'], det_full_ec_version(ec)) msg = "Filename '%s' of parsed easyconfig matches expected filename '%s'" % (spec, expected_fn) self.assertEqual(os.path.basename(spec), expected_fn, msg) name, easyblock = fetch_parameters_from_easyconfig(ec.rawtxt, ['name', 'easyblock']) # make sure easyconfig file is in expected location expected_subdir = os.path.join('easybuild', 'easyconfigs', letter_dir_for(name), name) subdir = os.path.join(*spec.split(os.path.sep)[-5:-1]) fail_msg = "Easyconfig file %s not in expected subdirectory %s" % (spec, expected_subdir) self.assertEqual(expected_subdir, subdir, fail_msg) # sanity check for software name, moduleclass self.assertEqual(ec['name'], name) self.assertTrue(ec['moduleclass'] in build_option('valid_module_classes')) # instantiate easyblock with easyconfig file app_class = get_easyblock_class(easyblock, name=name) # check that automagic fallback to ConfigureMake isn't done (deprecated behaviour) fn = os.path.basename(spec) error_msg = "%s relies on automagic fallback to ConfigureMake, should use easyblock = 'ConfigureMake' instead" % fn self.assertTrue(easyblock or not app_class is ConfigureMake, error_msg) app = app_class(ec) # more sanity checks self.assertTrue(name, app.name) self.assertTrue(ec['version'], app.version) # make sure that $root is not used, since it is not compatible with module files in Lua syntax res = re.findall('.*\$root.*', ec.rawtxt, re.M) error_msg = "Found use of '$root', not compatible with modules in Lua syntax, use '%%(installdir)s' instead: %s" self.assertFalse(res, error_msg % res) # make sure old GitHub urls for EasyBuild that include 'hpcugent' are no longer used old_urls = [ 'github.com/hpcugent/easybuild', 'hpcugent.github.com/easybuild', 'hpcugent.github.io/easybuild', ] for old_url in old_urls: self.assertFalse(old_url in ec.rawtxt, "Old URL '%s' not found in %s" % (old_url, spec)) # make sure binutils is included as a build dep if toolchain is GCCcore if ec['toolchain']['name'] == 'GCCcore': # with 'Tarball' easyblock: only unpacking, no building; Eigen is also just a tarball requires_binutils = ec['easyblock'] not in ['Tarball'] and ec['name'] not in ['Eigen'] # let's also exclude the very special case where the system GCC is used as GCCcore, and only apply this # exception to the dependencies of binutils (since we should eventually build a new binutils with GCCcore) if ec['toolchain']['version'] == 'system': binutils_complete_dependencies = ['M4', 'Bison', 'flex', 'help2man', 'zlib', 'binutils'] requires_binutils &= bool(ec['name'] not in binutils_complete_dependencies) # if no sources/extensions/components are specified, it's just a bundle (nothing is being compiled) requires_binutils &= bool(ec['sources'] or ec['exts_list'] or ec.get('components')) if requires_binutils: dep_names = [d['name'] for d in ec['builddependencies']] self.assertTrue('binutils' in dep_names, "binutils is a build dep in %s: %s" % (spec, dep_names)) # make sure all patch files are available specdir = os.path.dirname(spec) specfn = os.path.basename(spec) for patch in ec['patches']: if isinstance(patch, (tuple, list)): patch = patch[0] # only check actual patch files, not other files being copied via the patch functionality if patch.endswith('.patch'): patch_full = os.path.join(specdir, patch) msg = "Patch file %s is available for %s" % (patch_full, specfn) self.assertTrue(os.path.isfile(patch_full), msg) ext_patches = [] for ext in ec['exts_list']: if isinstance(ext, (tuple, list)) and len(ext) == 3: self.assertTrue(isinstance(ext[2], dict), "3rd element of extension spec is a dictionary") for ext_patch in ext[2].get('patches', []): if isinstance(ext_patch, (tuple, list)): ext_patch = ext_patch[0] # only check actual patch files, not other files being copied via the patch functionality if ext_patch.endswith('.patch'): ext_patch_full = os.path.join(specdir, ext_patch) msg = "Patch file %s is available for %s" % (ext_patch_full, specfn) self.assertTrue(os.path.isfile(ext_patch_full), msg) # check whether all extra_options defined for used easyblock are defined extra_opts = app.extra_options() for key in extra_opts: self.assertTrue(key in app.cfg) app.close_log() os.remove(app.logfile) # dump the easyconfig file handle, test_ecfile = tempfile.mkstemp() os.close(handle) ec.dump(test_ecfile) dumped_ec = EasyConfigParser(test_ecfile).get_config_dict() os.remove(test_ecfile) # inject dummy values for templates that are only known at a later stage dummy_template_values = { 'builddir': '/dummy/builddir', 'installdir': '/dummy/installdir', } ec.template_values.update(dummy_template_values) ec_dict = ec.parser.get_config_dict() orig_toolchain = ec_dict['toolchain'] for key in ec_dict: # skip parameters for which value is equal to default value orig_val = ec_dict[key] if key in DEFAULT_CONFIG and orig_val == DEFAULT_CONFIG[key][0]: continue if key in extra_opts and orig_val == extra_opts[key][0]: continue if key not in DEFAULT_CONFIG and key not in extra_opts: continue orig_val = resolve_template(ec_dict[key], ec.template_values) dumped_val = resolve_template(dumped_ec[key], ec.template_values) # take into account that dumped value for *dependencies may include hard-coded subtoolchains # if no easyconfig was found for the dependency with the 'parent' toolchain, # if may get resolved using a subtoolchain, which is then hardcoded in the dumped easyconfig if key in DEPENDENCY_PARAMETERS: # number of dependencies should remain the same self.assertEqual(len(orig_val), len(dumped_val)) for orig_dep, dumped_dep in zip(orig_val, dumped_val): # name/version should always match self.assertEqual(orig_dep[:2], dumped_dep[:2]) # 3rd value is versionsuffix; if len(dumped_dep) >= 3: # if no versionsuffix was specified in original dep spec, then dumped value should be empty string if len(orig_dep) >= 3: self.assertEqual(dumped_dep[2], orig_dep[2]) else: self.assertEqual(dumped_dep[2], '') # 4th value is toolchain spec if len(dumped_dep) >= 4: if len(orig_dep) >= 4: self.assertEqual(dumped_dep[3], orig_dep[3]) else: # if a subtoolchain is specifed (only) in the dumped easyconfig, # it should *not* be the same as the parent toolchain self.assertNotEqual(dumped_dep[3], (orig_toolchain['name'], orig_toolchain['version'])) else: self.assertEqual(orig_val, dumped_val) # cache the parsed easyconfig, to avoid that it is parsed again self.parsed_easyconfigs.append(ecs[0]) # test passed, so set back to True single_tests_ok = True and prev_single_tests_ok