def test_get_easyblock_class(self): """Test get_easyblock_class function.""" from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.easyblocks.generic.toolchain import Toolchain from easybuild.easyblocks.toy import EB_toy for easyblock, easyblock_class in [ ('ConfigureMake', ConfigureMake), ('easybuild.easyblocks.generic.configuremake.ConfigureMake', ConfigureMake), ('Toolchain', Toolchain), ('EB_toy', EB_toy), ]: self.assertEqual(get_easyblock_class(easyblock), easyblock_class) self.assertEqual(get_easyblock_class(None, name='gzip'), ConfigureMake) self.assertEqual(get_easyblock_class(None, name='toy'), EB_toy)
def process_easyconfig_file(ec_file): """Process an easyconfig file: fix if it's broken, back it up before fixing it inline (if requested).""" ectxt = read_file(ec_file) name, easyblock = fetch_parameters_from_easyconfig(ectxt, ['name', 'easyblock']) derived_easyblock_class = get_easyblock_class( easyblock, name=name, error_on_missing_easyblock=False) fixed_ectxt = fix_broken_easyconfig(ectxt, derived_easyblock_class) if ectxt != fixed_ectxt: if go.options.backup: try: backup_ec_file = '%s.bk' % ec_file i = 1 while os.path.exists(backup_ec_file): backup_ec_file = '%s.bk%d' % (ec_file, i) i += 1 os.rename(ec_file, backup_ec_file) log.info("Backed up %s to %s" % (ec_file, backup_ec_file)) except OSError, err: raise EasyBuildError( "Failed to backup %s before rewriting it: %s", ec_file, err) write_file(ec_file, fixed_ectxt) log.debug("Contents of fixed easyconfig file: %s" % fixed_ectxt) log.info("%s: fixed" % ec_file)
def avail_easyconfig_params(self): """ Print the available easyconfig parameters, for the given easyblock. """ app = get_easyblock_class(self.options.easyblock) extra = app.extra_options() mapping = convert_to_help(extra, has_default=False) if len(extra) > 0: ebb_msg = " (* indicates specific for the %s EasyBlock)" % app.__name__ extra_names = [x[0] for x in extra] else: ebb_msg = '' extra_names = [] txt = ["Available easyconfig parameters%s" % ebb_msg] params = [(k, v) for (k, v) in mapping.items() if k.upper() not in ['HIDDEN']] for key, values in params: txt.append("%s" % key.upper()) txt.append('-' * len(key)) for name, value in values: tabs = "\t" * (3 - (len(name) + 1) / 8) if name in extra_names: starred = '(*)' else: starred = '' txt.append("%s%s:%s%s" % (name, starred, tabs, value)) txt.append('') return "\n".join(txt)
def pre_configure_hook(self, *args, **kwargs): "Modify configopts (here is more efficient than parse_hook since only called once)" orig_enable_templating = self.cfg.enable_templating self.cfg.enable_templating = False modify_all_opts(self.cfg, opts_changes, opts_to_skip=PARSE_OPTS) ec = self.cfg # additional changes for CMakeMake EasyBlocks CMakeMake_configopts_changes = ( ' -DZLIB_ROOT=$NIXUSER_PROFILE ' + ' -DOPENGL_INCLUDE_DIR=$NIXUSER_PROFILE/include -DOPENGL_gl_LIBRARY=$NIXUSER_PROFILE/lib/libGL.so ' + ' -DOPENGL_glu_LIBRARY=$NIXUSER_PROFILE/lib/libGLU.so ' + ' -DJPEG_INCLUDE_DIR=$NIXUSER_PROFILE/include -DJPEG_LIBRARY=$NIXUSER_PROFILE/lib/libjpeg.so ' + ' -DPNG_PNG_INCLUDE_DIR=$NIXUSER_PROFILE/include -DPNG_LIBRARY=$NIXUSER_PROFILE/lib/libpng.so ' + ' -DPYTHON_EXECUTABLE=$EBROOTPYTHON/bin/python ' + ' -DCURL_LIBRARY=$NIXUSER_PROFILE/lib/libcurl.so -DCURL_INCLUDE_DIR=$NIXUSER_PROFILE/include ' + ' -DCMAKE_SYSTEM_PREFIX_PATH=$NIXUSER_PROFILE ' + ' -DCMAKE_SKIP_INSTALL_RPATH=ON ') if ec.easyblock is None or isinstance(ec.easyblock, str): c = get_easyblock_class(ec.easyblock, name=ec.name) elif isinstance(ec.easyblock, type): c = ec.easyblock if c == CMakeMake or issubclass(c, CMakeMake): # skip for those if (ec['name'], ec['version']) in [('ROOT', '5.34.36'), ('mariadb', '10.4.11')]: pass else: update_opts(ec, CMakeMake_configopts_changes, 'configopts', PREPEND) self.cfg.enable_templating = orig_enable_templating
def check_sha256_checksums(ecs, whitelist=None): """ Check whether all provided (parsed) easyconfigs have SHA256 checksums for sources & patches. :param whitelist: list of regex patterns on easyconfig filenames; check is skipped for matching easyconfigs :return: list of strings describing checksum issues (missing checksums, wrong checksum type, etc.) """ checksum_issues = [] if whitelist is None: whitelist = [] for ec in ecs: # skip whitelisted software ec_fn = os.path.basename(ec.path) if any(re.match(regex, ec_fn) for regex in whitelist): _log.info( "Skipping SHA256 checksum check for %s because of whitelist (%s)", ec.path, whitelist) continue eb_class = get_easyblock_class(ec['easyblock'], name=ec['name']) checksum_issues.extend(eb_class(ec).check_checksums()) return checksum_issues
def pre_configure_hook(self, *args, **kwargs): "Modify configopts (here is more efficient than parse_hook since only called once)" orig_enable_templating = self.cfg.enable_templating self.cfg.enable_templating = False modify_all_opts(self.cfg, opts_changes, opts_to_skip=PARSE_OPTS + ['exts_list', 'postinstallcmds', 'modluafooter']) # additional changes for CMakeMake EasyBlocks ec = self.cfg if ec.easyblock is None or isinstance(ec.easyblock, str): c = get_easyblock_class(ec.easyblock, name=ec.name) elif isinstance(ec.easyblock, type): c = ec.easyblock if c == CMakeMake or issubclass(c, CMakeMake): # skip for those if (ec['name'], ec['version']) in [('ROOT', '5.34.36'), ('mariadb', '10.4.11')]: pass else: update_opts(ec, ' -DCMAKE_SKIP_INSTALL_RPATH=ON ', 'configopts', PREPEND) # disable XHOST update_opts(ec, ' -DENABLE_XHOST=OFF ', 'configopts', PREPEND) # use verbose makefile to get the command lines that are executed update_opts(ec, ' -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON ', 'configopts', PREPEND) # additional changes for MesonNinja EasyBlocks if (c == MesonNinja or issubclass(c, MesonNinja)) and c != CMakeNinja: update_opts(ec, False, 'fail_on_missing_ninja_meson_dep', REPLACE) self.cfg.enable_templating = orig_enable_templating
def process_easyconfig_file(ec_file): """Process an easyconfig file: fix if it's broken, back it up before fixing it inline (if requested).""" ectxt = read_file(ec_file) name, easyblock = fetch_parameters_from_easyconfig(ectxt, ['name', 'easyblock']) derived_easyblock_class = get_easyblock_class(easyblock, name=name, default_fallback=False) fixed_ectxt = fix_broken_easyconfig(ectxt, derived_easyblock_class) if ectxt != fixed_ectxt: if go.options.backup: try: backup_ec_file = '%s.bk' % ec_file i = 1 while os.path.exists(backup_ec_file): backup_ec_file = '%s.bk%d' % (ec_file, i) i += 1 os.rename(ec_file, backup_ec_file) log.info("Backed up %s to %s" % (ec_file, backup_ec_file)) except OSError, err: raise EasyBuildError("Failed to backup %s before rewriting it: %s", ec_file, err) write_file(ec_file, fixed_ectxt) log.debug("Contents of fixed easyconfig file: %s" % fixed_ectxt) log.info("%s: fixed" % ec_file)
def dump_env_script(easyconfigs): """ Dump source scripts that set up build environment for specified easyconfigs. :param easyconfigs: list of easyconfigs to generate scripts for """ ecs_and_script_paths = [] for easyconfig in easyconfigs: script_path = '%s.env' % os.path.splitext(os.path.basename(easyconfig['spec']))[0] ecs_and_script_paths.append((easyconfig['ec'], script_path)) # don't just overwrite existing scripts existing_scripts = [s for (_, s) in ecs_and_script_paths if os.path.exists(s)] if existing_scripts: if build_option('force'): _log.info("Found existing scripts, overwriting them: %s", ' '.join(existing_scripts)) else: raise EasyBuildError("Script(s) already exists, not overwriting them (unless --force is used): %s", ' '.join(existing_scripts)) orig_env = copy.deepcopy(os.environ) for ec, script_path in ecs_and_script_paths: # obtain EasyBlock instance app_class = get_easyblock_class(ec['easyblock'], name=ec['name']) app = app_class(ec) # mimic dry run, and keep quiet app.dry_run = app.silent = app.toolchain.dry_run = True # prepare build environment (in dry run mode) app.check_readiness_step() app.prepare_step(start_dir=False) # compose script ecfile = os.path.basename(ec.path) script_lines = [ "#!/bin/bash", "# script to set up build environment as defined by EasyBuild v%s for %s" % (EASYBUILD_VERSION, ecfile), "# usage: source %s" % os.path.basename(script_path), ] script_lines.extend(['', "# toolchain & dependency modules"]) if app.toolchain.modules: script_lines.extend(["module load %s" % mod for mod in app.toolchain.modules]) else: script_lines.append("# (no modules loaded)") script_lines.extend(['', "# build environment"]) if app.toolchain.vars: env_vars = sorted(app.toolchain.vars.items()) script_lines.extend(["export %s='%s'" % (var, val.replace("'", "\\'")) for (var, val) in env_vars]) else: script_lines.append("# (no build environment defined)") write_file(script_path, '\n'.join(script_lines)) print_msg("Script to set up build environment for %s dumped to %s" % (ecfile, script_path), prefix=False) restore_env(orig_env)
def dump_env_script(easyconfigs): """ Dump source scripts that set up build environment for specified easyconfigs. :param easyconfigs: list of easyconfigs to generate scripts for """ ecs_and_script_paths = [] for easyconfig in easyconfigs: script_path = '%s.env' % os.path.splitext(os.path.basename(easyconfig['spec']))[0] ecs_and_script_paths.append((easyconfig['ec'], script_path)) # don't just overwrite existing scripts existing_scripts = [s for (_, s) in ecs_and_script_paths if os.path.exists(s)] if existing_scripts: if build_option('force'): _log.info("Found existing scripts, overwriting them: %s", ' '.join(existing_scripts)) else: raise EasyBuildError("Script(s) already exists, not overwriting them (unless --force is used): %s", ' '.join(existing_scripts)) orig_env = copy.deepcopy(os.environ) for ec, script_path in ecs_and_script_paths: # obtain EasyBlock instance app_class = get_easyblock_class(ec['easyblock'], name=ec['name']) app = app_class(ec) # mimic dry run, and keep quiet app.dry_run = app.silent = app.toolchain.dry_run = True # prepare build environment (in dry run mode) app.check_readiness_step() app.prepare_step(start_dir=False) # compose script ecfile = os.path.basename(ec.path) script_lines = [ "#!/bin/bash", "# script to set up build environment as defined by EasyBuild v%s for %s" % (EASYBUILD_VERSION, ecfile), "# usage: source %s" % os.path.basename(script_path), ] script_lines.extend(['', "# toolchain & dependency modules"]) if app.toolchain.modules: script_lines.extend(["module load %s" % mod for mod in app.toolchain.modules]) else: script_lines.append("# (no modules loaded)") script_lines.extend(['', "# build environment"]) if app.toolchain.vars: env_vars = sorted(app.toolchain.vars.items()) script_lines.extend(["export %s='%s'" % (var, val.replace("'", "\\'")) for (var, val) in env_vars]) else: script_lines.append("# (no build environment defined)") write_file(script_path, '\n'.join(script_lines)) print_msg("Script to set up build environment for %s dumped to %s" % (ecfile, script_path), prefix=False) restore_env(orig_env)
def test_get_easyblock_class(self): """Test get_easyblock_class function.""" from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.easyblocks.generic.toolchain import Toolchain from easybuild.easyblocks.toy import EB_toy for easyblock, easyblock_class in [ ('ConfigureMake', ConfigureMake), ('easybuild.easyblocks.generic.configuremake.ConfigureMake', ConfigureMake), ('Toolchain', Toolchain), ('EB_toy', EB_toy), ]: self.assertEqual(get_easyblock_class(easyblock), easyblock_class) self.assertEqual(get_easyblock_class(None, name='gzip', default_fallback=False), None) self.assertEqual(get_easyblock_class(None, name='toy'), EB_toy) self.assertErrorRegex(EasyBuildError, "Failed to import EB_TOY", get_easyblock_class, None, name='TOY') self.assertEqual(get_easyblock_class(None, name='TOY', error_on_failed_import=False), None)
def test_get_easyblock_class(self): """Test get_easyblock_class function.""" from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.easyblocks.generic.toolchain import Toolchain from easybuild.easyblocks.toy import EB_toy for easyblock, easyblock_class in [ ('ConfigureMake', ConfigureMake), ('easybuild.easyblocks.generic.configuremake.ConfigureMake', ConfigureMake), ('Toolchain', Toolchain), ('EB_toy', EB_toy), ]: self.assertEqual(get_easyblock_class(easyblock), easyblock_class) self.assertEqual(get_easyblock_class(None, name='gzip', default_fallback=False), None) self.assertEqual(get_easyblock_class(None, name='toy'), EB_toy) self.assertErrorRegex(EasyBuildError, "Failed to import EB_TOY", get_easyblock_class, None, name='TOY') self.assertEqual(get_easyblock_class(None, name='TOY', error_on_failed_import=False), None)
def template_module_only_test(self, easyblock, name='foo', version='1.3.2', extra_txt=''): """Test whether all easyblocks are compatible with --module-only.""" class_regex = re.compile("^class (.*)\(.*", re.M) self.log.debug("easyblock: %s" % easyblock) # read easyblock Python module f = open(easyblock, "r") txt = f.read() f.close() # obtain easyblock class name using regex res = class_regex.search(txt) if res: ebname = res.group(1) self.log.debug("Found class name for easyblock %s: %s" % (easyblock, ebname)) # figure out list of mandatory variables, and define with dummy values as necessary app_class = get_easyblock_class(ebname) # extend easyconfig to make sure mandatory custom easyconfig paramters are defined extra_options = app_class.extra_options() for (key, val) in extra_options.items(): if val[2] == MANDATORY: extra_txt += '%s = "foo"\n' % key # write easyconfig file self.writeEC(ebname, name=name, version=version, extratxt=extra_txt) # initialize easyblock # if this doesn't fail, the test succeeds app = app_class(EasyConfig(self.eb_file)) # run all steps, most should be skipped orig_workdir = os.getcwd() try: app.run_all_steps(run_test_cases=False) finally: os.chdir(orig_workdir) modfile = os.path.join(TMPDIR, 'modules', 'all', 'foo', '1.3.2') luamodfile = '%s.lua' % modfile self.assertTrue( os.path.exists(modfile) or os.path.exists(luamodfile), "Module file %s or %s was generated" % (modfile, luamodfile)) # cleanup app.close_log() os.remove(app.logfile) else: self.assertTrue(False, "Class found in easyblock %s" % easyblock)
def install_step(self): """Install components, if specified.""" comp_cnt = len(self.cfg['components']) for idx, cfg in enumerate(self.comp_cfgs): easyblock = cfg.get('easyblock') or self.cfg['default_easyblock'] if easyblock is None: raise EasyBuildError( "No easyblock specified for component %s v%s", cfg['name'], cfg['version']) elif easyblock == 'Bundle': raise EasyBuildError( "The '%s' easyblock can not be used to install components in a bundle", easyblock) print_msg("installing bundle component %s v%s (%d/%d)..." % (cfg['name'], cfg['version'], idx + 1, comp_cnt)) self.log.info("Installing component %s v%s using easyblock %s", cfg['name'], cfg['version'], easyblock) comp = get_easyblock_class(easyblock, name=cfg['name'])(cfg) # correct build/install dirs comp.builddir = self.builddir comp.install_subdir, comp.installdir = self.install_subdir, self.installdir # figure out correct start directory comp.guess_start_dir() # need to run fetch_patches to ensure per-component patches are applied comp.fetch_patches() # location of first unpacked source is used to determine where to apply patch(es) comp.src = [{'finalpath': comp.cfg['start_dir']}] # run relevant steps for step_name in ['patch', 'configure', 'build', 'install']: if step_name in cfg['skipsteps']: comp.log.info("Skipping '%s' step for component %s v%s", step_name, cfg['name'], cfg['version']) else: comp.run_step( step_name, [lambda x: getattr(x, '%s_step' % step_name)]) # update environment to ensure stuff provided by former components can be picked up by latter components # once the installation is finalised, this is handled by the generated module reqs = comp.make_module_req_guess() for envvar in reqs: curr_val = os.getenv(envvar, '') curr_paths = curr_val.split(os.pathsep) for subdir in reqs[envvar]: path = os.path.join(self.installdir, subdir) if path not in curr_paths: if curr_val: new_val = '%s:%s' % (path, curr_val) else: new_val = path env.setvar(envvar, new_val)
def template_init_test(self, easyblock): """Test whether all easyconfigs can be initialized.""" def check_extra_options_format(extra_options): """Make sure extra_options value is of correct format.""" # EasyBuild v1.x self.assertTrue(isinstance(extra_options, list)) for extra_option in extra_options: self.assertTrue(isinstance(extra_option, tuple)) self.assertEqual(len(extra_option), 2) self.assertTrue(isinstance(extra_option[0], basestring)) self.assertTrue(isinstance(extra_option[1], list)) self.assertEqual(len(extra_option[1]), 3) # EasyBuild v2.0 (breaks backward compatibility compared to v1.x) #self.assertTrue(isinstance(extra_options, dict)) #for key in extra_options: # self.assertTrue(isinstance(extra_options[key], list)) # self.assertTrue(len(extra_options[key]), 3) class_regex = re.compile("^class (.*)\(.*", re.M) self.log.debug("easyblock: %s" % easyblock) # obtain easyblock class name using regex f = open(easyblock, "r") txt = f.read() f.close() res = class_regex.search(txt) if res: ebname = res.group(1) self.log.debug("Found class name for easyblock %s: %s" % (easyblock, ebname)) # figure out list of mandatory variables, and define with dummy values as necessary app_class = get_easyblock_class(ebname) extra_options = app_class.extra_options() check_extra_options_format(extra_options) # extend easyconfig to make sure mandatory custom easyconfig paramters are defined extra_txt = '' for (key, val) in extra_options: if val[2] == MANDATORY: extra_txt += '%s = "foo"\n' % key # write easyconfig file self.writeEC(ebname, extra_txt) # initialize easyblock # if this doesn't fail, the test succeeds app = app_class(EasyConfig(self.eb_file)) # cleanup app.close_log() os.remove(app.logfile) else: self.assertTrue(False, "Class found in easyblock %s" % easyblock)
def template_init_test(self, easyblock): """Test whether all easyconfigs can be initialized.""" def check_extra_options_format(extra_options): """Make sure extra_options value is of correct format.""" # EasyBuild v1.x self.assertTrue(isinstance(extra_options, list)) for extra_option in extra_options: self.assertTrue(isinstance(extra_option, tuple)) self.assertEqual(len(extra_option), 2) self.assertTrue(isinstance(extra_option[0], basestring)) self.assertTrue(isinstance(extra_option[1], list)) self.assertEqual(len(extra_option[1]), 3) # EasyBuild v2.0 (breaks backward compatibility compared to v1.x) #self.assertTrue(isinstance(extra_options, dict)) #for key in extra_options: # self.assertTrue(isinstance(extra_options[key], list)) # self.assertTrue(len(extra_options[key]), 3) class_regex = re.compile("^class (.*)\(.*", re.M) self.log.debug("easyblock: %s" % easyblock) # obtain easyblock class name using regex f = open(easyblock, "r") txt = f.read() f.close() res = class_regex.search(txt) if res: ebname = res.group(1) self.log.debug("Found class name for easyblock %s: %s" % (easyblock, ebname)) # figure out list of mandatory variables, and define with dummy values as necessary app_class = get_easyblock_class(ebname) extra_options = app_class.extra_options() check_extra_options_format(extra_options) # extend easyconfig to make sure mandatory custom easyconfig paramters are defined extra_txt = '' for (key, val) in extra_options: if val[2] == MANDATORY: extra_txt += '%s = "foo"\n' % key # write easyconfig file self.writeEC(ebname, extra_txt) # initialize easyblock # if this doesn't fail, the test succeeds app = app_class(EasyConfig(self.eb_file)) # cleanup app.close_log() os.remove(app.logfile) else: self.assertTrue(False, "Class found in easyblock %s" % easyblock)
def test_make_module_pythonpackage(self): """Test make_module_step of PythonPackage easyblock.""" app_class = get_easyblock_class('PythonPackage') self.writeEC('PythonPackage', name='testpypkg', version='3.14') app = app_class(EasyConfig(self.eb_file)) # install dir should not be there yet self.assertFalse(os.path.exists(app.installdir)) # create install dir and populate it with subdirs/files mkdir(app.installdir, parents=True) # $PATH, $LD_LIBRARY_PATH, $LIBRARY_PATH, $CPATH, $PKG_CONFIG_PATH write_file(os.path.join(app.installdir, 'bin', 'foo'), 'echo foo!') write_file(os.path.join(app.installdir, 'include', 'foo.h'), 'bar') write_file(os.path.join(app.installdir, 'lib', 'libfoo.a'), 'libfoo') pyver = '.'.join(map(str, sys.version_info[:2])) write_file(os.path.join(app.installdir, 'lib', 'python%s' % pyver, 'site-packages', 'foo.egg'), 'foo egg') write_file(os.path.join(app.installdir, 'lib64', 'pkgconfig', 'foo.pc'), 'libfoo: foo') # create module file app.make_module_step() self.assertTrue(TMPDIR in app.installdir) self.assertTrue(TMPDIR in app.installdir_mod) modtxt = None for cand_mod_filename in ['3.14', '3.14.lua']: full_modpath = os.path.join(app.installdir_mod, 'testpypkg', cand_mod_filename) if os.path.exists(full_modpath): modtxt = read_file(full_modpath) break self.assertFalse(modtxt is None) regexs = [ (r'^prepend.path.*\WCPATH\W.*include"?\W*$', True), (r'^prepend.path.*\WLD_LIBRARY_PATH\W.*lib"?\W*$', True), (r'^prepend.path.*\WLIBRARY_PATH\W.*lib"?\W*$', True), (r'^prepend.path.*\WPATH\W.*bin"?\W*$', True), (r'^prepend.path.*\WPKG_CONFIG_PATH\W.*lib64/pkgconfig"?\W*$', True), (r'^prepend.path.*\WPYTHONPATH\W.*lib/python2.[0-9]/site-packages"?\W*$', True), # lib64 doesn't contain any library files, so these are *not* included in $LD_LIBRARY_PATH or $LIBRARY_PATH (r'^prepend.path.*\WLD_LIBRARY_PATH\W.*lib64', False), (r'^prepend.path.*\WLIBRARY_PATH\W.*lib64', False), ] for (pattern, found) in regexs: regex = re.compile(pattern, re.M) if found: assert_msg = "Pattern '%s' found in: %s" % (regex.pattern, modtxt) else: assert_msg = "Pattern '%s' not found in: %s" % (regex.pattern, modtxt) self.assertEqual(bool(regex.search(modtxt)), found, assert_msg)
def test_make_module_pythonpackage(self): """Test make_module_step of PythonPackage easyblock.""" app_class = get_easyblock_class('PythonPackage') self.writeEC('PythonPackage', name='testpypkg', version='3.14') app = app_class(EasyConfig(self.eb_file)) # install dir should not be there yet self.assertFalse(os.path.exists(app.installdir)) # create install dir and populate it with subdirs/files mkdir(app.installdir, parents=True) # $PATH, $LD_LIBRARY_PATH, $LIBRARY_PATH, $CPATH, $PKG_CONFIG_PATH write_file(os.path.join(app.installdir, 'bin', 'foo'), 'echo foo!') write_file(os.path.join(app.installdir, 'include', 'foo.h'), 'bar') write_file(os.path.join(app.installdir, 'lib', 'libfoo.a'), 'libfoo') write_file(os.path.join(app.installdir, 'lib', 'python2.7', 'site-packages', 'foo.egg'), 'foo egg') write_file(os.path.join(app.installdir, 'lib64', 'pkgconfig', 'foo.pc'), 'libfoo: foo') # create module file app.make_module_step() self.assertTrue(TMPDIR in app.installdir) self.assertTrue(TMPDIR in app.installdir_mod) modtxt = None for cand_mod_filename in ['3.14', '3.14.lua']: full_modpath = os.path.join(app.installdir_mod, 'testpypkg', cand_mod_filename) if os.path.exists(full_modpath): modtxt = read_file(full_modpath) break self.assertFalse(modtxt is None) regexs = [ (r'^prepend.path.*\WCPATH\W.*include"?\W*$', True), (r'^prepend.path.*\WLD_LIBRARY_PATH\W.*lib"?\W*$', True), (r'^prepend.path.*\WLIBRARY_PATH\W.*lib"?\W*$', True), (r'^prepend.path.*\WPATH\W.*bin"?\W*$', True), (r'^prepend.path.*\WPKG_CONFIG_PATH\W.*lib64/pkgconfig"?\W*$', True), (r'^prepend.path.*\WPYTHONPATH\W.*lib/python2.7/site-packages"?\W*$', True), # lib64 doesn't contain any library files, so these are *not* included in $LD_LIBRARY_PATH or $LIBRARY_PATH (r'^prepend.path.*\WLD_LIBRARY_PATH\W.*lib64', False), (r'^prepend.path.*\WLIBRARY_PATH\W.*lib64', False), ] for (pattern, found) in regexs: regex = re.compile(pattern, re.M) if found: assert_msg = "Pattern '%s' found in: %s" % (regex.pattern, modtxt) else: assert_msg = "Pattern '%s' not found in: %s" % (regex.pattern, modtxt) self.assertEqual(bool(regex.search(modtxt)), found, assert_msg)
def template_module_only_test(self, easyblock, name='foo', version='1.3.2', extra_txt=''): """Test whether all easyblocks are compatible with --module-only.""" class_regex = re.compile("^class (.*)\(.*", re.M) self.log.debug("easyblock: %s" % easyblock) # read easyblock Python module f = open(easyblock, "r") txt = f.read() f.close() # obtain easyblock class name using regex res = class_regex.search(txt) if res: ebname = res.group(1) self.log.debug("Found class name for easyblock %s: %s" % (easyblock, ebname)) # figure out list of mandatory variables, and define with dummy values as necessary app_class = get_easyblock_class(ebname) # extend easyconfig to make sure mandatory custom easyconfig paramters are defined extra_options = app_class.extra_options() for (key, val) in extra_options.items(): if val[2] == MANDATORY: extra_txt += '%s = "foo"\n' % key # write easyconfig file self.writeEC(ebname, name=name, version=version, extratxt=extra_txt) # initialize easyblock # if this doesn't fail, the test succeeds app = app_class(EasyConfig(self.eb_file)) # run all steps, most should be skipped orig_workdir = os.getcwd() try: app.run_all_steps(run_test_cases=False) finally: os.chdir(orig_workdir) modfile = os.path.join(TMPDIR, 'modules', 'all', 'foo', '1.3.2') luamodfile = '%s.lua' % modfile self.assertTrue(os.path.exists(modfile) or os.path.exists(luamodfile), "Module file %s or %s was generated" % (modfile, luamodfile)) # cleanup app.close_log() os.remove(app.logfile) else: self.assertTrue(False, "Class found in easyblock %s" % easyblock)
def install_step(self): """Install components, if specified.""" comp_cnt = len(self.cfg['components']) for idx, cfg in enumerate(self.comp_cfgs): easyblock = cfg.get('easyblock') or self.cfg['default_easyblock'] if easyblock is None: raise EasyBuildError( "No easyblock specified for component %d v%d", cfg['name'], cfg['version']) elif easyblock == 'Bundle': raise EasyBuildError( "The '%s' easyblock can not be used to install components in a bundle", easyblock) print_msg("installing bundle component %s v%s (%d/%d)..." % (cfg['name'], cfg['version'], idx + 1, comp_cnt)) self.log.info("Installing component %s v%s using easyblock %s", cfg['name'], cfg['version'], easyblock) comp = get_easyblock_class(easyblock, name=cfg['name'])(cfg) # correct build/install dirs comp.builddir = self.builddir comp.install_subdir, comp.installdir = self.install_subdir, self.installdir # figure out correct start directory comp.guess_start_dir() # run relevant steps comp.patch_step() comp.configure_step() comp.build_step() comp.install_step() # update environment to ensure stuff provided by former components can be picked up by latter components # once the installation is finalised, this is handled by the generated module reqs = comp.make_module_req_guess() for envvar in reqs: curr_val = os.getenv(envvar, '') curr_paths = curr_val.split(os.pathsep) for subdir in reqs[envvar]: path = os.path.join(self.installdir, subdir) if path not in curr_paths: if curr_val: new_val = '%s:%s' % (path, curr_val) else: new_val = path env.setvar(envvar, new_val)
def install_step(self): """Install components, if specified.""" comp_cnt = len(self.cfg['components']) for idx, cfg in enumerate(self.comp_cfgs): easyblock = cfg.get('easyblock') or self.cfg['default_easyblock'] if easyblock is None: raise EasyBuildError("No easyblock specified for component %s v%s", cfg['name'], cfg['version']) elif easyblock == 'Bundle': raise EasyBuildError("The '%s' easyblock can not be used to install components in a bundle", easyblock) print_msg("installing bundle component %s v%s (%d/%d)..." % (cfg['name'], cfg['version'], idx+1, comp_cnt)) self.log.info("Installing component %s v%s using easyblock %s", cfg['name'], cfg['version'], easyblock) comp = get_easyblock_class(easyblock, name=cfg['name'])(cfg) # correct build/install dirs comp.builddir = self.builddir comp.install_subdir, comp.installdir = self.install_subdir, self.installdir # figure out correct start directory comp.guess_start_dir() # need to run fetch_patches to ensure per-component patches are applied comp.fetch_patches() # location of first unpacked source is used to determine where to apply patch(es) comp.src = [{'finalpath': comp.cfg['start_dir']}] # run relevant steps for step_name in ['patch', 'configure', 'build', 'install']: if step_name in cfg['skipsteps']: comp.log.info("Skipping '%s' step for component %s v%s", step_name, cfg['name'], cfg['version']) else: comp.run_step(step_name, [lambda x: getattr(x, '%s_step' % step_name)]) # update environment to ensure stuff provided by former components can be picked up by latter components # once the installation is finalised, this is handled by the generated module reqs = comp.make_module_req_guess() for envvar in reqs: curr_val = os.getenv(envvar, '') curr_paths = curr_val.split(os.pathsep) for subdir in reqs[envvar]: path = os.path.join(self.installdir, subdir) if path not in curr_paths: if curr_val: new_val = '%s:%s' % (path, curr_val) else: new_val = path env.setvar(envvar, new_val)
def avail_easyconfig_params(easyblock, output_format): """ Compose overview of available easyconfig parameters, in specified format. """ params = copy.deepcopy(DEFAULT_CONFIG) # include list of extra parameters (if any) extra_params = {} app = get_easyblock_class(easyblock, default_fallback=False) if app is not None: extra_params = app.extra_options() params.update(extra_params) # compose title title = "Available easyconfig parameters" if extra_params: title += " (* indicates specific to the %s easyblock)" % app.__name__ # group parameters by category grouped_params = OrderedDict() for category in sorted_categories(): # exclude hidden parameters if category[1].upper() in [HIDDEN]: continue grpname = category[1] grouped_params[grpname] = {} for name, (dflt, descr, cat) in params.items(): if cat == category: if name in extra_params: # mark easyblock-specific parameters name = '%s*' % name grouped_params[grpname].update({name: (descr, dflt)}) if not grouped_params[grpname]: del grouped_params[grpname] # compose output, according to specified format (txt, rst, ...) avail_easyconfig_params_functions = { FORMAT_RST: avail_easyconfig_params_rst, FORMAT_TXT: avail_easyconfig_params_txt, } return avail_easyconfig_params_functions[output_format](title, grouped_params)
def dump_env_script(easyconfigs): """ Dump source scripts that set up build environment for specified easyconfigs. :param easyconfigs: list of easyconfigs to generate scripts for """ ecs_and_script_paths = [] for easyconfig in easyconfigs: script_path = '%s.env' % os.path.splitext( os.path.basename(easyconfig['spec']))[0] ecs_and_script_paths.append((easyconfig['ec'], script_path)) # don't just overwrite existing scripts existing_scripts = [ s for (_, s) in ecs_and_script_paths if os.path.exists(s) ] if existing_scripts: if build_option('force'): _log.info("Found existing scripts, overwriting them: %s", ' '.join(existing_scripts)) else: raise EasyBuildError( "Script(s) already exists, not overwriting them (unless --force is used): %s", ' '.join(existing_scripts)) orig_env = copy.deepcopy(os.environ) for ec, script_path in ecs_and_script_paths: # obtain EasyBlock instance app_class = get_easyblock_class(ec['easyblock'], name=ec['name']) app = app_class(ec) # mimic dry run, and keep quiet app.dry_run = app.silent = app.toolchain.dry_run = True # prepare build environment (in dry run mode) app.check_readiness_step() app.prepare_step(start_dir=False) # create the environment dump dump_env_easyblock(app, orig_env=orig_env, ec_path=ec.path, script_path=script_path)
def avail_easyconfig_params(easyblock, output_format): """ Compose overview of available easyconfig parameters, in specified format. """ params = copy.deepcopy(DEFAULT_CONFIG) # include list of extra parameters (if any) extra_params = {} app = get_easyblock_class(easyblock, default_fallback=False) if app is not None: extra_params = app.extra_options() params.update(extra_params) # compose title title = "Available easyconfig parameters" if extra_params: title += " (* indicates specific to the %s easyblock)" % app.__name__ # group parameters by category grouped_params = OrderedDict() for category in sorted_categories(): # exclude hidden parameters if category[1].upper() in [HIDDEN]: continue grpname = category[1] grouped_params[grpname] = {} for name, (dflt, descr, cat) in params.items(): if cat == category: if name in extra_params: # mark easyblock-specific parameters name = '%s*' % name grouped_params[grpname].update({name: (descr, dflt)}) if not grouped_params[grpname]: del grouped_params[grpname] # compose output, according to specified format (txt, rst, ...) avail_easyconfig_params_functions = { FORMAT_RST: avail_easyconfig_params_rst, FORMAT_TXT: avail_easyconfig_params_txt, } return avail_easyconfig_params_functions[output_format](title, grouped_params)
def install_step(self): """Install components, if specified.""" comp_cnt = len(self.cfg['components']) for idx, cfg in enumerate(self.comp_cfgs): easyblock = cfg.get('easyblock') or self.cfg['default_easyblock'] if easyblock is None: raise EasyBuildError("No easyblock specified for component %d v%d", cfg['name'], cfg['version']) elif easyblock == 'Bundle': raise EasyBuildError("The '%s' easyblock can not be used to install components in a bundle", easyblock) print_msg("installing bundle component %s v%s (%d/%d)..." % (cfg['name'], cfg['version'], idx+1, comp_cnt)) self.log.info("Installing component %s v%s using easyblock %s", cfg['name'], cfg['version'], easyblock) comp = get_easyblock_class(easyblock, name=cfg['name'])(cfg) # correct build/install dirs comp.builddir = self.builddir comp.install_subdir, comp.installdir = self.install_subdir, self.installdir # figure out correct start directory comp.guess_start_dir() # run relevant steps comp.patch_step() comp.configure_step() comp.build_step() comp.install_step() # update environment to ensure stuff provided by former components can be picked up by latter components # once the installation is finalised, this is handled by the generated module reqs = comp.make_module_req_guess() for envvar in reqs: curr_val = os.getenv(envvar, '') curr_paths = curr_val.split(os.pathsep) for subdir in reqs[envvar]: path = os.path.join(self.installdir, subdir) if path not in curr_paths: if curr_val: new_val = '%s:%s' % (path, curr_val) else: new_val = path env.setvar(envvar, new_val)
def check_sha256_checksums(ecs, whitelist=None): """ Check whether all provided (parsed) easyconfigs have SHA256 checksums for sources & patches. :param whitelist: list of regex patterns on easyconfig filenames; check is skipped for matching easyconfigs :return: list of strings describing checksum issues (missing checksums, wrong checksum type, etc.) """ checksum_issues = [] if whitelist is None: whitelist = [] for ec in ecs: # skip whitelisted software ec_fn = os.path.basename(ec.path) if any(re.match(regex, ec_fn) for regex in whitelist): _log.info("Skipping SHA256 checksum check for %s because of whitelist (%s)", ec.path, whitelist) continue eb_class = get_easyblock_class(ec['easyblock'], name=ec['name']) checksum_issues.extend(eb_class(ec).check_checksums()) return checksum_issues
def template_easyconfig_test(self, spec): """Tests for an individual easyconfig: parsing, instantiating easyblock, check patches, ...""" # set to False, so it's False in case of this test failing global single_tests_ok prev_single_tests_ok = single_tests_ok single_tests_ok = False # parse easyconfig ecs = process_easyconfig(spec) if len(ecs) == 1: ec = ecs[0]['ec'] else: self.assertTrue(False, "easyconfig %s does not contain blocks, yields only one parsed easyconfig" % spec) # check easyconfig file name expected_fn = '%s-%s.eb' % (ec['name'], det_full_ec_version(ec)) msg = "Filename '%s' of parsed easyconfig matches expected filename '%s'" % (spec, expected_fn) self.assertEqual(os.path.basename(spec), expected_fn, msg) name, easyblock = fetch_parameters_from_easyconfig(ec.rawtxt, ['name', 'easyblock']) # make sure easyconfig file is in expected location expected_subdir = os.path.join('easybuild', 'easyconfigs', letter_dir_for(name), name) subdir = os.path.join(*spec.split(os.path.sep)[-5:-1]) fail_msg = "Easyconfig file %s not in expected subdirectory %s" % (spec, expected_subdir) self.assertEqual(expected_subdir, subdir, fail_msg) # sanity check for software name, moduleclass self.assertEqual(ec['name'], name) self.assertTrue(ec['moduleclass'] in build_option('valid_module_classes')) # instantiate easyblock with easyconfig file app_class = get_easyblock_class(easyblock, name=name) # check that automagic fallback to ConfigureMake isn't done (deprecated behaviour) fn = os.path.basename(spec) error_msg = "%s relies on automagic fallback to ConfigureMake, should use easyblock = 'ConfigureMake' instead" % fn self.assertTrue(easyblock or not app_class is ConfigureMake, error_msg) app = app_class(ec) # more sanity checks self.assertTrue(name, app.name) self.assertTrue(ec['version'], app.version) # make sure all patch files are available specdir = os.path.dirname(spec) specfn = os.path.basename(spec) for patch in ec['patches']: if isinstance(patch, (tuple, list)): patch = patch[0] # only check actual patch files, not other files being copied via the patch functionality if patch.endswith('.patch'): patch_full = os.path.join(specdir, patch) msg = "Patch file %s is available for %s" % (patch_full, specfn) self.assertTrue(os.path.isfile(patch_full), msg) ext_patches = [] for ext in ec['exts_list']: if isinstance(ext, (tuple, list)) and len(ext) == 3: self.assertTrue(isinstance(ext[2], dict), "3rd element of extension spec is a dictionary") for ext_patch in ext[2].get('patches', []): if isinstance(ext_patch, (tuple, list)): ext_patch = ext_patch[0] # only check actual patch files, not other files being copied via the patch functionality if ext_patch.endswith('.patch'): ext_patch_full = os.path.join(specdir, ext_patch) msg = "Patch file %s is available for %s" % (ext_patch_full, specfn) self.assertTrue(os.path.isfile(ext_patch_full), msg) # check whether all extra_options defined for used easyblock are defined extra_opts = app.extra_options() for key in extra_opts: self.assertTrue(key in app.cfg) app.close_log() os.remove(app.logfile) # dump the easyconfig file handle, test_ecfile = tempfile.mkstemp() os.close(handle) ec.dump(test_ecfile) dumped_ec = EasyConfigParser(test_ecfile).get_config_dict() os.remove(test_ecfile) # inject dummy values for templates that are only known at a later stage dummy_template_values = { 'builddir': '/dummy/builddir', 'installdir': '/dummy/installdir', } ec.template_values.update(dummy_template_values) ec_dict = ec.parser.get_config_dict() orig_toolchain = ec_dict['toolchain'] for key in ec_dict: # skip parameters for which value is equal to default value orig_val = ec_dict[key] if key in DEFAULT_CONFIG and orig_val == DEFAULT_CONFIG[key][0]: continue if key in extra_opts and orig_val == extra_opts[key][0]: continue if key not in DEFAULT_CONFIG and key not in extra_opts: continue orig_val = resolve_template(ec_dict[key], ec.template_values) dumped_val = resolve_template(dumped_ec[key], ec.template_values) # take into account that dumped value for *dependencies may include hard-coded subtoolchains # if no easyconfig was found for the dependency with the 'parent' toolchain, # if may get resolved using a subtoolchain, which is then hardcoded in the dumped easyconfig if key in DEPENDENCY_PARAMETERS: # number of dependencies should remain the same self.assertEqual(len(orig_val), len(dumped_val)) for orig_dep, dumped_dep in zip(orig_val, dumped_val): # name/version should always match self.assertEqual(orig_dep[:2], dumped_dep[:2]) # 3rd value is versionsuffix; if len(dumped_dep) >= 3: # if no versionsuffix was specified in original dep spec, then dumped value should be empty string if len(orig_dep) >= 3: self.assertEqual(dumped_dep[2], orig_dep[2]) else: self.assertEqual(dumped_dep[2], '') # 4th value is toolchain spec if len(dumped_dep) >= 4: if len(orig_dep) >= 4: self.assertEqual(dumped_dep[3], orig_dep[3]) else: # if a subtoolchain is specifed (only) in the dumped easyconfig, # it should *not* be the same as the parent toolchain self.assertNotEqual(dumped_dep[3], (orig_toolchain['name'], orig_toolchain['version'])) else: self.assertEqual(orig_val, dumped_val) # cache the parsed easyconfig, to avoid that it is parsed again self.parsed_easyconfigs.append(ecs[0]) # test passed, so set back to True single_tests_ok = True and prev_single_tests_ok
def template_init_test(self, easyblock, name='foo', version='1.3.2'): """Test whether all easyblocks can be initialized.""" def check_extra_options_format(extra_options): """Make sure extra_options value is of correct format.""" # EasyBuild v2.0: dict with <string> keys and <list> values self.assertTrue(isinstance(extra_options, dict)) extra_options.items() extra_options.keys() extra_options.values() for key in extra_options.keys(): self.assertTrue(isinstance(extra_options[key], list)) self.assertTrue(len(extra_options[key]), 3) class_regex = re.compile("^class (.*)\(.*", re.M) self.log.debug("easyblock: %s" % easyblock) # read easyblock Python module f = open(easyblock, "r") txt = f.read() f.close() # make sure error reporting is done correctly (no more log.error, log.exception) log_method_regexes = [ re.compile(r"log\.error\("), re.compile(r"log\.exception\("), re.compile(r"log\.raiseException\("), ] for regex in log_method_regexes: self.assertFalse( regex.search(txt), "No match for '%s' in %s" % (regex.pattern, easyblock)) # obtain easyblock class name using regex res = class_regex.search(txt) if res: ebname = res.group(1) self.log.debug("Found class name for easyblock %s: %s" % (easyblock, ebname)) # figure out list of mandatory variables, and define with dummy values as necessary app_class = get_easyblock_class(ebname) extra_options = app_class.extra_options() check_extra_options_format(extra_options) # extend easyconfig to make sure mandatory custom easyconfig paramters are defined extra_txt = '' for (key, val) in extra_options.items(): if val[2] == MANDATORY: extra_txt += '%s = "foo"\n' % key # write easyconfig file self.writeEC(ebname, name=name, version=version, extratxt=extra_txt) # initialize easyblock # if this doesn't fail, the test succeeds app = app_class(EasyConfig(self.eb_file)) # check whether easyblock instance is still using functions from a deprecated location mod = __import__(app.__module__, [], [], ['easybuild.easyblocks']) moved_functions = [ 'modify_env', 'parse_log_for_error', 'read_environment', 'run_cmd', 'run_cmd_qa' ] for fn in moved_functions: if hasattr(mod, fn): tup = (fn, app.__module__, globals()[fn].__module__) self.assertTrue( getattr(mod, fn) is globals()[fn], "%s in %s is imported from %s" % tup) renamed_functions = [ ('source_paths', 'source_path'), ('get_avail_core_count', 'get_core_count'), ('get_os_type', 'get_kernel_name'), ('det_full_ec_version', 'det_installversion'), ] for (new_fn, old_fn) in renamed_functions: self.assertFalse( hasattr(mod, old_fn), "%s: %s is replaced by %s" % (app.__module__, old_fn, new_fn)) # cleanup app.close_log() os.remove(app.logfile) else: self.assertTrue(False, "Class found in easyblock %s" % easyblock)
def template_easyconfig_test(self, spec): """Tests for an individual easyconfig: parsing, instantiating easyblock, check patches, ...""" # set to False, so it's False in case of this test failing global single_tests_ok prev_single_tests_ok = single_tests_ok single_tests_ok = False # parse easyconfig ecs = process_easyconfig(spec) if len(ecs) == 1: ec = ecs[0]['ec'] else: self.assertTrue(False, "easyconfig %s does not contain blocks, yields only one parsed easyconfig" % spec) # check easyconfig file name expected_fn = '%s-%s.eb' % (ec['name'], det_full_ec_version(ec)) msg = "Filename '%s' of parsed easconfig matches expected filename '%s'" % (spec, expected_fn) self.assertEqual(os.path.basename(spec), expected_fn, msg) name, easyblock = fetch_parameters_from_easyconfig(ec.rawtxt, ['name', 'easyblock']) # sanity check for software name self.assertTrue(ec['name'], name) # instantiate easyblock with easyconfig file app_class = get_easyblock_class(easyblock, name=name) # check that automagic fallback to ConfigureMake isn't done (deprecated behaviour) fn = os.path.basename(spec) error_msg = "%s relies on automagic fallback to ConfigureMake, should use easyblock = 'ConfigureMake' instead" % fn self.assertTrue(easyblock or not app_class is ConfigureMake, error_msg) app = app_class(ec) # more sanity checks self.assertTrue(name, app.name) self.assertTrue(ec['version'], app.version) # make sure all patch files are available specdir = os.path.dirname(spec) specfn = os.path.basename(spec) for patch in ec['patches']: if isinstance(patch, (tuple, list)): patch = patch[0] # only check actual patch files, not other files being copied via the patch functionality if patch.endswith('.patch'): patch_full = os.path.join(specdir, patch) msg = "Patch file %s is available for %s" % (patch_full, specfn) self.assertTrue(os.path.isfile(patch_full), msg) ext_patches = [] for ext in ec['exts_list']: if isinstance(ext, (tuple, list)) and len(ext) == 3: self.assertTrue(isinstance(ext[2], dict), "3rd element of extension spec is a dictionary") for ext_patch in ext[2].get('patches', []): if isinstance(ext_patch, (tuple, list)): ext_patch = ext_patch[0] # only check actual patch files, not other files being copied via the patch functionality if ext_patch.endswith('.patch'): ext_patch_full = os.path.join(specdir, ext_patch) msg = "Patch file %s is available for %s" % (ext_patch_full, specfn) self.assertTrue(os.path.isfile(ext_patch_full), msg) # check whether all extra_options defined for used easyblock are defined for key in app.extra_options(): self.assertTrue(key in app.cfg) app.close_log() os.remove(app.logfile) # cache the parsed easyconfig, to avoid that it is parsed again self.parsed_easyconfigs.append(ecs[0]) # test passed, so set back to True single_tests_ok = True and prev_single_tests_ok
def template_easyconfig_test(self, spec): """Tests for an individual easyconfig: parsing, instantiating easyblock, check patches, ...""" # set to False, so it's False in case of this test failing global single_tests_ok prev_single_tests_ok = single_tests_ok single_tests_ok = False # parse easyconfig ecs = process_easyconfig(spec) if len(ecs) == 1: ec = ecs[0]['ec'] else: self.assertTrue(False, "easyconfig %s does not contain blocks, yields only one parsed easyconfig" % spec) # check easyconfig file name expected_fn = '%s-%s.eb' % (ec['name'], det_full_ec_version(ec)) msg = "Filename '%s' of parsed easyconfig matches expected filename '%s'" % (spec, expected_fn) self.assertEqual(os.path.basename(spec), expected_fn, msg) name, easyblock = fetch_parameters_from_easyconfig(ec.rawtxt, ['name', 'easyblock']) # make sure easyconfig file is in expected location expected_subdir = os.path.join('easybuild', 'easyconfigs', letter_dir_for(name), name) subdir = os.path.join(*spec.split(os.path.sep)[-5:-1]) fail_msg = "Easyconfig file %s not in expected subdirectory %s" % (spec, expected_subdir) self.assertEqual(expected_subdir, subdir, fail_msg) # sanity check for software name, moduleclass self.assertEqual(ec['name'], name) self.assertTrue(ec['moduleclass'] in build_option('valid_module_classes')) # instantiate easyblock with easyconfig file app_class = get_easyblock_class(easyblock, name=name) # check that automagic fallback to ConfigureMake isn't done (deprecated behaviour) fn = os.path.basename(spec) error_msg = "%s relies on automagic fallback to ConfigureMake, should use easyblock = 'ConfigureMake' instead" % fn self.assertTrue(easyblock or not app_class is ConfigureMake, error_msg) app = app_class(ec) # more sanity checks self.assertTrue(name, app.name) self.assertTrue(ec['version'], app.version) # make sure that $root is not used, since it is not compatible with module files in Lua syntax res = re.findall('.*\$root.*', ec.rawtxt, re.M) error_msg = "Found use of '$root', not compatible with modules in Lua syntax, use '%%(installdir)s' instead: %s" self.assertFalse(res, error_msg % res) # make sure old GitHub urls for EasyBuild that include 'hpcugent' are no longer used old_urls = [ 'github.com/hpcugent/easybuild', 'hpcugent.github.com/easybuild', 'hpcugent.github.io/easybuild', ] for old_url in old_urls: self.assertFalse(old_url in ec.rawtxt, "Old URL '%s' not found in %s" % (old_url, spec)) # make sure binutils is included as a build dep if toolchain is GCCcore if ec['toolchain']['name'] == 'GCCcore': # with 'Tarball' easyblock: only unpacking, no building; Eigen is also just a tarball requires_binutils = ec['easyblock'] not in ['Tarball'] and ec['name'] not in ['Eigen'] # let's also exclude the very special case where the system GCC is used as GCCcore, and only apply this # exception to the dependencies of binutils (since we should eventually build a new binutils with GCCcore) if ec['toolchain']['version'] == 'system': binutils_complete_dependencies = ['M4', 'Bison', 'flex', 'help2man', 'zlib', 'binutils'] requires_binutils &= bool(ec['name'] not in binutils_complete_dependencies) # if no sources/extensions/components are specified, it's just a bundle (nothing is being compiled) requires_binutils &= bool(ec['sources'] or ec['exts_list'] or ec.get('components')) if requires_binutils: dep_names = [d['name'] for d in ec['builddependencies']] self.assertTrue('binutils' in dep_names, "binutils is a build dep in %s: %s" % (spec, dep_names)) # make sure all patch files are available specdir = os.path.dirname(spec) specfn = os.path.basename(spec) for patch in ec['patches']: if isinstance(patch, (tuple, list)): patch = patch[0] # only check actual patch files, not other files being copied via the patch functionality if patch.endswith('.patch'): patch_full = os.path.join(specdir, patch) msg = "Patch file %s is available for %s" % (patch_full, specfn) self.assertTrue(os.path.isfile(patch_full), msg) ext_patches = [] for ext in ec['exts_list']: if isinstance(ext, (tuple, list)) and len(ext) == 3: self.assertTrue(isinstance(ext[2], dict), "3rd element of extension spec is a dictionary") for ext_patch in ext[2].get('patches', []): if isinstance(ext_patch, (tuple, list)): ext_patch = ext_patch[0] # only check actual patch files, not other files being copied via the patch functionality if ext_patch.endswith('.patch'): ext_patch_full = os.path.join(specdir, ext_patch) msg = "Patch file %s is available for %s" % (ext_patch_full, specfn) self.assertTrue(os.path.isfile(ext_patch_full), msg) # check whether all extra_options defined for used easyblock are defined extra_opts = app.extra_options() for key in extra_opts: self.assertTrue(key in app.cfg) app.close_log() os.remove(app.logfile) # dump the easyconfig file handle, test_ecfile = tempfile.mkstemp() os.close(handle) ec.dump(test_ecfile) dumped_ec = EasyConfigParser(test_ecfile).get_config_dict() os.remove(test_ecfile) # inject dummy values for templates that are only known at a later stage dummy_template_values = { 'builddir': '/dummy/builddir', 'installdir': '/dummy/installdir', } ec.template_values.update(dummy_template_values) ec_dict = ec.parser.get_config_dict() orig_toolchain = ec_dict['toolchain'] for key in ec_dict: # skip parameters for which value is equal to default value orig_val = ec_dict[key] if key in DEFAULT_CONFIG and orig_val == DEFAULT_CONFIG[key][0]: continue if key in extra_opts and orig_val == extra_opts[key][0]: continue if key not in DEFAULT_CONFIG and key not in extra_opts: continue orig_val = resolve_template(ec_dict[key], ec.template_values) dumped_val = resolve_template(dumped_ec[key], ec.template_values) # take into account that dumped value for *dependencies may include hard-coded subtoolchains # if no easyconfig was found for the dependency with the 'parent' toolchain, # if may get resolved using a subtoolchain, which is then hardcoded in the dumped easyconfig if key in DEPENDENCY_PARAMETERS: # number of dependencies should remain the same self.assertEqual(len(orig_val), len(dumped_val)) for orig_dep, dumped_dep in zip(orig_val, dumped_val): # name/version should always match self.assertEqual(orig_dep[:2], dumped_dep[:2]) # 3rd value is versionsuffix; if len(dumped_dep) >= 3: # if no versionsuffix was specified in original dep spec, then dumped value should be empty string if len(orig_dep) >= 3: self.assertEqual(dumped_dep[2], orig_dep[2]) else: self.assertEqual(dumped_dep[2], '') # 4th value is toolchain spec if len(dumped_dep) >= 4: if len(orig_dep) >= 4: self.assertEqual(dumped_dep[3], orig_dep[3]) else: # if a subtoolchain is specifed (only) in the dumped easyconfig, # it should *not* be the same as the parent toolchain self.assertNotEqual(dumped_dep[3], (orig_toolchain['name'], orig_toolchain['version'])) else: self.assertEqual(orig_val, dumped_val) # cache the parsed easyconfig, to avoid that it is parsed again self.parsed_easyconfigs.append(ecs[0]) # test passed, so set back to True single_tests_ok = True and prev_single_tests_ok
def template_init_test(self, easyblock): """Test whether all easyblocks can be initialized.""" def check_extra_options_format(extra_options): """Make sure extra_options value is of correct format.""" # EasyBuild v2.0: dict with <string> keys and <list> values self.assertTrue(isinstance(extra_options, dict)) extra_options.items() extra_options.keys() extra_options.values() for key in extra_options.keys(): self.assertTrue(isinstance(extra_options[key], list)) self.assertTrue(len(extra_options[key]), 3) class_regex = re.compile("^class (.*)\(.*", re.M) self.log.debug("easyblock: %s" % easyblock) # read easyblock Python module f = open(easyblock, "r") txt = f.read() f.close() # make sure error reporting is done correctly (no more log.error, log.exception) log_method_regexes = [ re.compile(r"log\.error\("), re.compile(r"log\.exception\("), re.compile(r"log\.raiseException\("), ] for regex in log_method_regexes: self.assertFalse(regex.search(txt), "No match for '%s' in %s" % (regex.pattern, easyblock)) # obtain easyblock class name using regex res = class_regex.search(txt) if res: ebname = res.group(1) self.log.debug("Found class name for easyblock %s: %s" % (easyblock, ebname)) # figure out list of mandatory variables, and define with dummy values as necessary app_class = get_easyblock_class(ebname) extra_options = app_class.extra_options() check_extra_options_format(extra_options) # extend easyconfig to make sure mandatory custom easyconfig paramters are defined extra_txt = '' for (key, val) in extra_options.items(): if val[2] == MANDATORY: extra_txt += '%s = "foo"\n' % key # write easyconfig file self.writeEC(ebname, extra_txt) # initialize easyblock # if this doesn't fail, the test succeeds app = app_class(EasyConfig(self.eb_file)) # check whether easyblock instance is still using functions from a deprecated location mod = __import__(app.__module__, [], [], ['easybuild.easyblocks']) moved_functions = ['modify_env', 'parse_log_for_error', 'read_environment', 'run_cmd', 'run_cmd_qa'] for fn in moved_functions: if hasattr(mod, fn): tup = (fn, app.__module__, globals()[fn].__module__) self.assertTrue(getattr(mod, fn) is globals()[fn], "%s in %s is imported from %s" % tup) renamed_functions = [ ('source_paths', 'source_path'), ('get_avail_core_count', 'get_core_count'), ('get_os_type', 'get_kernel_name'), ('det_full_ec_version', 'det_installversion'), ] for (new_fn, old_fn) in renamed_functions: self.assertFalse(hasattr(mod, old_fn), "%s: %s is replaced by %s" % (app.__module__, old_fn, new_fn)) # cleanup app.close_log() os.remove(app.logfile) else: self.assertTrue(False, "Class found in easyblock %s" % easyblock)
subfolders.remove('.git') for ec_file in files: if not ec_file.endswith('.eb') or ec_file in ["TEMPLATE.eb"]: log.warning("SKIPPING %s/%s" % (root, ec_file)) continue ec_file = join(root, ec_file) ec_file = read(ec_file) try: ec = EasyConfig(ec_file) log.info("found valid easyconfig %s" % ec) if not ec.name in names: log.info("found new software package %s" % ec.name) ec.easyblock = None # check if an easyblock exists ebclass = get_easyblock_class(None, name=ec.name, error_on_missing_easyblock=False) if ebclass is not None: module = ebclass.__module__.split('.')[-1] if module != "configuremake": ec.easyblock = module configs.append(ec) names.append(ec.name) except Exception, err: raise EasyBuildError("faulty easyconfig %s: %s", ec_file, err) log.info("Found easyconfigs: %s" % [x.name for x in configs]) # sort by name configs = sorted(configs, key=lambda config: config.name.lower()) firstl = ""
def template_easyconfig_test(self, spec): """Tests for an individual easyconfig: parsing, instantiating easyblock, check patches, ...""" # set to False, so it's False in case of this test failing global single_tests_ok prev_single_tests_ok = single_tests_ok single_tests_ok = False # parse easyconfig ecs = process_easyconfig(spec) if len(ecs) == 1: ec = ecs[0]['ec'] # cache the parsed easyconfig, to avoid that it is parsed again self.parsed_easyconfigs.append(ecs[0]) else: self.assertTrue( False, "easyconfig %s does not contain blocks, yields only one parsed easyconfig" % spec) # check easyconfig file name expected_fn = '%s-%s.eb' % (ec['name'], det_full_ec_version(ec)) msg = "Filename '%s' of parsed easyconfig matches expected filename '%s'" % ( spec, expected_fn) self.assertEqual(os.path.basename(spec), expected_fn, msg) name, easyblock = fetch_parameters_from_easyconfig(ec.rawtxt, ['name', 'easyblock']) # make sure easyconfig file is in expected location expected_subdir = os.path.join('easybuild', 'easyconfigs', letter_dir_for(name), name) subdir = os.path.join(*spec.split(os.path.sep)[-5:-1]) fail_msg = "Easyconfig file %s not in expected subdirectory %s" % ( spec, expected_subdir) self.assertEqual(expected_subdir, subdir, fail_msg) # sanity check for software name, moduleclass self.assertEqual(ec['name'], name) self.assertTrue(ec['moduleclass'] in build_option('valid_module_classes')) # instantiate easyblock with easyconfig file app_class = get_easyblock_class(easyblock, name=name) # check that automagic fallback to ConfigureMake isn't done (deprecated behaviour) fn = os.path.basename(spec) error_msg = "%s relies on automagic fallback to ConfigureMake, should use easyblock = 'ConfigureMake' instead" % fn self.assertTrue(easyblock or app_class is not ConfigureMake, error_msg) app = app_class(ec) # more sanity checks self.assertTrue(name, app.name) self.assertTrue(ec['version'], app.version) # make sure that $root is not used, since it is not compatible with module files in Lua syntax res = re.findall('.*\$root.*', ec.rawtxt, re.M) error_msg = "Found use of '$root', not compatible with modules in Lua syntax, use '%%(installdir)s' instead: %s" self.assertFalse(res, error_msg % res) # make sure old GitHub urls for EasyBuild that include 'hpcugent' are no longer used old_urls = [ 'github.com/hpcugent/easybuild', 'hpcugent.github.com/easybuild', 'hpcugent.github.io/easybuild', ] for old_url in old_urls: self.assertFalse(old_url in ec.rawtxt, "Old URL '%s' not found in %s" % (old_url, spec)) # make sure binutils is included as a build dep if toolchain is GCCcore if ec['toolchain']['name'] == 'GCCcore': # with 'Tarball' easyblock: only unpacking, no building; Eigen is also just a tarball requires_binutils = ec['easyblock'] not in [ 'Tarball' ] and ec['name'] not in ['Eigen'] # let's also exclude the very special case where the system GCC is used as GCCcore, and only apply this # exception to the dependencies of binutils (since we should eventually build a new binutils with GCCcore) if ec['toolchain']['version'] == 'system': binutils_complete_dependencies = [ 'M4', 'Bison', 'flex', 'help2man', 'zlib', 'binutils' ] requires_binutils &= bool( ec['name'] not in binutils_complete_dependencies) # if no sources/extensions/components are specified, it's just a bundle (nothing is being compiled) requires_binutils &= bool(ec['sources'] or ec['exts_list'] or ec.get('components')) if requires_binutils: dep_names = [d['name'] for d in ec.builddependencies()] self.assertTrue( 'binutils' in dep_names, "binutils is a build dep in %s: %s" % (spec, dep_names)) # make sure all patch files are available specdir = os.path.dirname(spec) specfn = os.path.basename(spec) for patch in ec['patches']: if isinstance(patch, (tuple, list)): patch = patch[0] # only check actual patch files, not other files being copied via the patch functionality if patch.endswith('.patch'): patch_full = os.path.join(specdir, patch) msg = "Patch file %s is available for %s" % (patch_full, specfn) self.assertTrue(os.path.isfile(patch_full), msg) for ext in ec['exts_list']: if isinstance(ext, (tuple, list)) and len(ext) == 3: self.assertTrue(isinstance(ext[2], dict), "3rd element of extension spec is a dictionary") for ext_patch in ext[2].get('patches', []): if isinstance(ext_patch, (tuple, list)): ext_patch = ext_patch[0] # only check actual patch files, not other files being copied via the patch functionality if ext_patch.endswith('.patch'): ext_patch_full = os.path.join(specdir, ext_patch) msg = "Patch file %s is available for %s" % ( ext_patch_full, specfn) self.assertTrue(os.path.isfile(ext_patch_full), msg) # check whether all extra_options defined for used easyblock are defined extra_opts = app.extra_options() for key in extra_opts: self.assertTrue(key in app.cfg) app.close_log() os.remove(app.logfile) # dump the easyconfig file handle, test_ecfile = tempfile.mkstemp() os.close(handle) ec.dump(test_ecfile) dumped_ec = EasyConfigParser(test_ecfile).get_config_dict() os.remove(test_ecfile) # inject dummy values for templates that are only known at a later stage dummy_template_values = { 'builddir': '/dummy/builddir', 'installdir': '/dummy/installdir', } ec.template_values.update(dummy_template_values) ec_dict = ec.parser.get_config_dict() orig_toolchain = ec_dict['toolchain'] for key in ec_dict: # skip parameters for which value is equal to default value orig_val = ec_dict[key] if key in DEFAULT_CONFIG and orig_val == DEFAULT_CONFIG[key][0]: continue if key in extra_opts and orig_val == extra_opts[key][0]: continue if key not in DEFAULT_CONFIG and key not in extra_opts: continue orig_val = resolve_template(ec_dict[key], ec.template_values) dumped_val = resolve_template(dumped_ec[key], ec.template_values) # take into account that dumped value for *dependencies may include hard-coded subtoolchains # if no easyconfig was found for the dependency with the 'parent' toolchain, # if may get resolved using a subtoolchain, which is then hardcoded in the dumped easyconfig if key in DEPENDENCY_PARAMETERS: # number of dependencies should remain the same self.assertEqual(len(orig_val), len(dumped_val)) for orig_dep, dumped_dep in zip(orig_val, dumped_val): # name/version should always match self.assertEqual(orig_dep[:2], dumped_dep[:2]) # 3rd value is versionsuffix; if len(dumped_dep) >= 3: # if no versionsuffix was specified in original dep spec, then dumped value should be empty string if len(orig_dep) >= 3: self.assertEqual(dumped_dep[2], orig_dep[2]) else: self.assertEqual(dumped_dep[2], '') # 4th value is toolchain spec if len(dumped_dep) >= 4: if len(orig_dep) >= 4: self.assertEqual(dumped_dep[3], orig_dep[3]) else: # if a subtoolchain is specifed (only) in the dumped easyconfig, # it should *not* be the same as the parent toolchain self.assertNotEqual(dumped_dep[3], (orig_toolchain['name'], orig_toolchain['version'])) else: self.assertEqual(orig_val, dumped_val) # test passed, so set back to True single_tests_ok = True and prev_single_tests_ok
if '.git' in subfolders: log.info("found .git subfolder, ignoring it") subfolders.remove('.git') for ec_file in files: if not ec_file.endswith('.eb') or ec_file in ["TEMPLATE.eb"]: log.warning("SKIPPING %s/%s" % (root, ec_file)) continue ec_file = join(root, ec_file) ec_file = read(ec_file) try: ec = EasyConfig(ec_file) log.info("found valid easyconfig %s" % ec) if not ec.name in names: log.info("found new software package %s" % ec) # check if an easyblock exists module = get_easyblock_class(None, name=ec.name).__module__.split('.')[-1] if module != "configuremake": ec.easyblock = module else: ec.easyblock = None configs.append(ec) names.append(ec.name) except Exception, err: log.error("faulty easyconfig %s: %s" % (ec_file, err)) log.info("Found easyconfigs: %s" % [x.name for x in configs]) # sort by name configs = sorted(configs, key=lambda config : config.name.lower()) firstl = "" # print out the configs in markdown format for the wiki
def template_easyconfig_test(self, spec): """Test whether all easyconfigs can be initialized.""" # set to False, so it's False in case of this test failing global single_tests_ok prev_single_tests_ok = single_tests_ok single_tests_ok = False # parse easyconfig ecs = process_easyconfig(spec) if len(ecs) == 1: ec = ecs[0]['ec'] else: self.assertTrue(False, "easyconfig %s does not contain blocks, yields only one parsed easyconfig" % spec) # sanity check for software name name = fetch_parameter_from_easyconfig_file(spec, 'name') self.assertTrue(ec['name'], name) # try and fetch easyblock spec from easyconfig easyblock = fetch_parameter_from_easyconfig_file(spec, 'easyblock') # instantiate easyblock with easyconfig file app_class = get_easyblock_class(easyblock, name=name) app = app_class(ec) # more sanity checks self.assertTrue(name, app.name) self.assertTrue(ec['version'], app.version) # make sure all patch files are available specdir = os.path.dirname(spec) specfn = os.path.basename(spec) for patch in ec['patches']: if isinstance(patch, (tuple, list)): patch = patch[0] # only check actual patch files, not other files being copied via the patch functionality if patch.endswith('.patch'): patch_full = os.path.join(specdir, patch) msg = "Patch file %s is available for %s" % (patch_full, specfn) self.assertTrue(os.path.isfile(patch_full), msg) ext_patches = [] for ext in ec['exts_list']: if isinstance(ext, (tuple, list)) and len(ext) == 3: self.assertTrue(isinstance(ext[2], dict), "3rd element of extension spec is a dictionary") for ext_patch in ext[2].get('patches', []): if isinstance(ext_patch, (tuple, list)): ext_patch = ext_patch[0] # only check actual patch files, not other files being copied via the patch functionality if ext_patch.endswith('.patch'): ext_patch_full = os.path.join(specdir, ext_patch) msg = "Patch file %s is available for %s" % (ext_patch_full, specfn) self.assertTrue(os.path.isfile(ext_patch_full), msg) app.close_log() os.remove(app.logfile) # cache the parsed easyconfig, to avoid that it is parsed again self.parsed_easyconfigs.append(ecs[0]) # test passed, so set back to True single_tests_ok = True and prev_single_tests_ok
def template_module_only_test(self, easyblock, name='foo', version='1.3.2', extra_txt=''): """Test whether all easyblocks are compatible with --module-only.""" tmpdir = tempfile.mkdtemp() class_regex = re.compile("^class (.*)\(.*", re.M) self.log.debug("easyblock: %s" % easyblock) # read easyblock Python module f = open(easyblock, "r") txt = f.read() f.close() # obtain easyblock class name using regex res = class_regex.search(txt) if res: ebname = res.group(1) self.log.debug("Found class name for easyblock %s: %s" % (easyblock, ebname)) toolchain = None # figure out list of mandatory variables, and define with dummy values as necessary app_class = get_easyblock_class(ebname) # easyblocks deriving from IntelBase require a license file to be found for --module-only if app_class == IntelBase or IntelBase in app_class.__bases__: os.environ['INTEL_LICENSE_FILE'] = os.path.join(tmpdir, 'intel.lic') write_file(os.environ['INTEL_LICENSE_FILE'], '# dummy license') if app_class == EB_IMOD: # $JAVA_HOME must be set for IMOD os.environ['JAVA_HOME'] = tmpdir if app_class == EB_OpenFOAM: # proper toolchain must be used for OpenFOAM(-Extend), to determine value to set for $WM_COMPILER write_file(os.path.join(tmpdir, 'GCC', '4.9.3-2.25'), '\n'.join([ '#%Module', 'setenv EBROOTGCC %s' % tmpdir, 'setenv EBVERSIONGCC 4.9.3', ])) write_file(os.path.join(tmpdir, 'OpenMPI', '1.10.2-GCC-4.9.3-2.25'), '\n'.join([ '#%Module', 'setenv EBROOTOPENMPI %s' % tmpdir, 'setenv EBVERSIONOPENMPI 1.10.2', ])) write_file(os.path.join(tmpdir, 'gompi', '2016a'), '\n'.join([ '#%Module', 'module load GCC/4.9.3-2.25', 'module load OpenMPI/1.10.2-GCC-4.9.3-2.25', ])) os.environ['MODULEPATH'] = tmpdir toolchain = {'name': 'gompi', 'version': '2016a'} # extend easyconfig to make sure mandatory custom easyconfig paramters are defined extra_options = app_class.extra_options() for (key, val) in extra_options.items(): if val[2] == MANDATORY: extra_txt += '%s = "foo"\n' % key # write easyconfig file self.writeEC(ebname, name=name, version=version, extratxt=extra_txt, toolchain=toolchain) # initialize easyblock # if this doesn't fail, the test succeeds app = app_class(EasyConfig(self.eb_file)) # run all steps, most should be skipped orig_workdir = os.getcwd() try: app.run_all_steps(run_test_cases=False) finally: os.chdir(orig_workdir) modfile = os.path.join(TMPDIR, 'modules', 'all', name, version) luamodfile = '%s.lua' % modfile self.assertTrue(os.path.exists(modfile) or os.path.exists(luamodfile), "Module file %s or %s was generated" % (modfile, luamodfile)) if os.path.exists(modfile): modtxt = read_file(modfile) else: modtxt = read_file(luamodfile) none_regex = re.compile('None') self.assertFalse(none_regex.search(modtxt), "None not found in module file: %s" % modtxt) # cleanup app.close_log() os.remove(app.logfile) shutil.rmtree(tmpdir) else: self.assertTrue(False, "Class found in easyblock %s" % easyblock)
def __init__(self, *args, **kwargs): """Initialize easyblock.""" super(Bundle, self).__init__(*args, **kwargs) self.altroot = None self.altversion = None # list of EasyConfig instances for components self.comp_cfgs = [] # list of EasyConfig instances of components for which to run sanity checks self.comp_cfgs_sanity_check = [] # list of sources for bundle itself *must* be empty if self.cfg['sources']: raise EasyBuildError( "List of sources for bundle itself must be empty, found %s", self.cfg['sources']) if self.cfg['patches']: raise EasyBuildError( "List of patches for bundle itself must be empty, found %s", self.cfg['patches']) # disable templating to avoid premature resolving of template values self.cfg.enable_templating = False # list of checksums for patches (must be included after checksums for sources) checksums_patches = [] if self.cfg['sanity_check_components'] and self.cfg[ 'sanity_check_all_components']: raise EasyBuildError( "sanity_check_components and sanity_check_all_components cannot be enabled together" ) # backup and reset general sanity checks from main body of ec, if component-specific sanity checks are enabled # necessary to avoid: # - duplicating the general sanity check across all components running sanity checks # - general sanity checks taking precedence over those defined in a component's easyblock self.backup_sanity_paths = self.cfg['sanity_check_paths'] self.backup_sanity_cmds = self.cfg['sanity_check_commands'] if self.cfg['sanity_check_components'] or self.cfg[ 'sanity_check_all_components']: # reset general sanity checks, to be restored later self.cfg['sanity_check_paths'] = {} self.cfg['sanity_check_commands'] = {} for comp in self.cfg['components']: comp_name, comp_version, comp_specs = comp[0], comp[1], {} if len(comp) == 3: comp_specs = comp[2] comp_cfg = self.cfg.copy() comp_cfg['name'] = comp_name comp_cfg['version'] = comp_version # determine easyblock to use for this component # - if an easyblock is specified explicitely, that will be used # - if not, a software-specific easyblock will be considered by get_easyblock_class # - if no easyblock was found, default_easyblock is considered comp_easyblock = comp_specs.get('easyblock') easyblock_class = get_easyblock_class( comp_easyblock, name=comp_name, error_on_missing_easyblock=False) if easyblock_class is None: if self.cfg['default_easyblock']: easyblock = self.cfg['default_easyblock'] easyblock_class = get_easyblock_class(easyblock) if easyblock_class is None: raise EasyBuildError( "No easyblock found for component %s v%s", comp_name, comp_version) else: self.log.info( "Using default easyblock %s for component %s", easyblock, comp_name) else: easyblock = easyblock_class.__name__ self.log.info("Using easyblock %s for component %s", easyblock, comp_name) if easyblock == 'Bundle': raise EasyBuildError( "The Bundle easyblock can not be used to install components in a bundle" ) comp_cfg.easyblock = easyblock_class # make sure that extra easyconfig parameters are known, so they can be set extra_opts = comp_cfg.easyblock.extra_options() comp_cfg.extend_params(copy.deepcopy(extra_opts)) comp_cfg.generate_template_values() # do not inherit easyblock to use from parent (since that would result in an infinite loop in install_step) comp_cfg['easyblock'] = None # reset list of sources/source_urls/checksums comp_cfg['sources'] = comp_cfg['source_urls'] = comp_cfg[ 'checksums'] = comp_cfg['patches'] = [] for key in self.cfg['default_component_specs']: comp_cfg[key] = self.cfg['default_component_specs'][key] for key in comp_specs: comp_cfg[key] = comp_specs[key] # enable resolving of templates for component-specific EasyConfig instance comp_cfg.enable_templating = True # 'sources' is strictly required if comp_cfg['sources']: # If per-component source URLs are provided, attach them directly to the relevant sources if comp_cfg['source_urls']: for source in comp_cfg['sources']: if isinstance(source, string_type): self.cfg.update( 'sources', [{ 'filename': source, 'source_urls': comp_cfg['source_urls'] }]) elif isinstance(source, dict): # Update source_urls in the 'source' dict to use the one for the components # (if it doesn't already exist) if 'source_urls' not in source: source['source_urls'] = comp_cfg['source_urls'] self.cfg.update('sources', [source]) else: raise EasyBuildError( "Source %s for component %s is neither a string nor a dict, cannot " "process it.", source, comp_cfg['name']) else: # add component sources to list of sources self.cfg.update('sources', comp_cfg['sources']) else: raise EasyBuildError( "No sources specification for component %s v%s", comp_name, comp_version) if comp_cfg['checksums']: src_cnt = len(comp_cfg['sources']) # add per-component checksums for sources to list of checksums self.cfg.update('checksums', comp_cfg['checksums'][:src_cnt]) # add per-component checksums for patches to list of checksums for patches checksums_patches.extend(comp_cfg['checksums'][src_cnt:]) if comp_cfg['patches']: self.cfg.update('patches', comp_cfg['patches']) self.comp_cfgs.append(comp_cfg) self.cfg.update('checksums', checksums_patches) self.cfg.enable_templating = True # restore general sanity checks if using component-specific sanity checks if self.cfg['sanity_check_components'] or self.cfg[ 'sanity_check_all_components']: self.cfg['sanity_check_paths'] = self.backup_sanity_paths self.cfg['sanity_check_commands'] = self.backup_sanity_cmds
subfolders.remove('.git') for ec_file in files: if not ec_file.endswith('.eb') or ec_file in ["TEMPLATE.eb"]: log.warning("SKIPPING %s/%s" % (root, ec_file)) continue ec_file = join(root, ec_file) ec_file = read(ec_file) try: ec = EasyConfig(ec_file) log.info("found valid easyconfig %s" % ec) if not ec.name in names: log.info("found new software package %s" % ec.name) ec.easyblock = None # check if an easyblock exists ebclass = get_easyblock_class(None, name=ec.name, default_fallback=False) if ebclass is not None: module = ebclass.__module__.split('.')[-1] if module != "configuremake": ec.easyblock = module configs.append(ec) names.append(ec.name) except Exception, err: raise EasyBuildError("faulty easyconfig %s: %s", ec_file, err) log.info("Found easyconfigs: %s" % [x.name for x in configs]) # sort by name configs = sorted(configs, key=lambda config: config.name.lower()) firstl = ""
def template_easyconfig_test(self, spec): """Test whether all easyconfigs can be initialized.""" # set to False, so it's False in case of this test failing global single_tests_ok prev_single_tests_ok = single_tests_ok single_tests_ok = False # parse easyconfig ecs = process_easyconfig(spec) if len(ecs) == 1: ec = ecs[0]['ec'] else: self.assertTrue( False, "easyconfig %s does not contain blocks, yields only one parsed easyconfig" % spec) # sanity check for software name name = fetch_parameter_from_easyconfig_file(spec, 'name') self.assertTrue(ec['name'], name) # try and fetch easyblock spec from easyconfig easyblock = fetch_parameter_from_easyconfig_file(spec, 'easyblock') # instantiate easyblock with easyconfig file app_class = get_easyblock_class(easyblock, name=name) app = app_class(ec) # more sanity checks self.assertTrue(name, app.name) self.assertTrue(ec['version'], app.version) # make sure all patch files are available specdir = os.path.dirname(spec) specfn = os.path.basename(spec) for patch in ec['patches']: if isinstance(patch, (tuple, list)): patch = patch[0] # only check actual patch files, not other files being copied via the patch functionality if patch.endswith('.patch'): patch_full = os.path.join(specdir, patch) msg = "Patch file %s is available for %s" % (patch_full, specfn) self.assertTrue(os.path.isfile(patch_full), msg) ext_patches = [] for ext in ec['exts_list']: if isinstance(ext, (tuple, list)) and len(ext) == 3: self.assertTrue(isinstance(ext[2], dict), "3rd element of extension spec is a dictionary") for ext_patch in ext[2].get('patches', []): if isinstance(ext_patch, (tuple, list)): ext_patch = ext_patch[0] # only check actual patch files, not other files being copied via the patch functionality if ext_patch.endswith('.patch'): ext_patch_full = os.path.join(specdir, ext_patch) msg = "Patch file %s is available for %s" % ( ext_patch_full, specfn) self.assertTrue(os.path.isfile(ext_patch_full), msg) app.close_log() os.remove(app.logfile) # cache the parsed easyconfig, to avoid that it is parsed again self.parsed_easyconfigs.append(ecs[0]) # test passed, so set back to True single_tests_ok = True and prev_single_tests_ok
def __init__(self, *args, **kwargs): """Initialize easyblock.""" super(Bundle, self).__init__(*args, **kwargs) self.altroot = None self.altversion = None # list of EasyConfig instances for components self.comp_cfgs = [] # list of sources for bundle itself *must* be empty if self.cfg['sources']: raise EasyBuildError( "List of sources for bundle itself must be empty, found %s", self.cfg['sources']) # disable templating to avoid premature resolving of template values self.cfg.enable_templating = False # list of checksums for patches (must be included after checksums for sources) checksums_patches = [] for comp in self.cfg['components']: comp_name, comp_version, comp_specs = comp[0], comp[1], {} if len(comp) == 3: comp_specs = comp[2] comp_cfg = self.cfg.copy() easyblock = comp_specs.get( 'easyblock') or self.cfg['default_easyblock'] if easyblock is None: raise EasyBuildError( "No easyblock specified for component %s v%s", comp_cfg['name'], comp_cfg['version']) elif easyblock == 'Bundle': raise EasyBuildError( "The Bundle easyblock can not be used to install components in a bundle" ) comp_cfg.easyblock = get_easyblock_class(easyblock, name=comp_cfg['name']) # make sure that extra easyconfig parameters are known, so they can be set extra_opts = comp_cfg.easyblock.extra_options() comp_cfg.extend_params(copy.deepcopy(extra_opts)) comp_cfg['name'] = comp_name comp_cfg['version'] = comp_version comp_cfg.generate_template_values() # do not inherit easyblock to use from parent (since that would result in an infinite loop in install_step) comp_cfg['easyblock'] = None # reset list of sources/source_urls/checksums comp_cfg['sources'] = comp_cfg['source_urls'] = comp_cfg[ 'checksums'] = [] for key in self.cfg['default_component_specs']: comp_cfg[key] = self.cfg['default_component_specs'][key] for key in comp_specs: comp_cfg[key] = comp_specs[key] # enable resolving of templates for component-specific EasyConfig instance comp_cfg.enable_templating = True # 'sources' is strictly required if comp_cfg['sources']: # If per-component source URLs are provided, attach them directly to the relevant sources if comp_cfg['source_urls']: for source in comp_cfg['sources']: if isinstance(source, basestring): self.cfg.update( 'sources', [{ 'filename': source, 'source_urls': comp_cfg['source_urls'] }]) elif isinstance(source, dict): # Update source_urls in the 'source' dict to use the one for the components # (if it doesn't already exist) if 'source_urls' not in source: source['source_urls'] = comp_cfg['source_urls'] self.cfg.update('sources', [source]) else: raise EasyBuildError( "Source %s for component %s is neither a string nor a dict, cannot " "process it.", source, comp_cfg['name']) else: # add component sources to list of sources self.cfg.update('sources', comp_cfg['sources']) else: raise EasyBuildError( "No sources specification for component %s v%s", comp_name, comp_version) if comp_cfg['checksums']: src_cnt = len(comp_cfg['sources']) # add per-component checksums for sources to list of checksums self.cfg.update('checksums', comp_cfg['checksums'][:src_cnt]) # add per-component checksums for patches to list of checksums for patches checksums_patches.extend(comp_cfg['checksums'][src_cnt:]) self.comp_cfgs.append(comp_cfg) self.cfg.update('checksums', checksums_patches) self.cfg.enable_templating = True
def test_make_module_pythonpackage(self): """Test make_module_step of PythonPackage easyblock.""" app_class = get_easyblock_class('PythonPackage') self.writeEC('PythonPackage', name='testpypkg', version='3.14') app = app_class(EasyConfig(self.eb_file)) # install dir should not be there yet self.assertFalse(os.path.exists(app.installdir), "%s should not exist" % app.installdir) # create install dir and populate it with subdirs/files mkdir(app.installdir, parents=True) # $PATH, $LD_LIBRARY_PATH, $LIBRARY_PATH, $CPATH, $PKG_CONFIG_PATH write_file(os.path.join(app.installdir, 'bin', 'foo'), 'echo foo!') write_file(os.path.join(app.installdir, 'include', 'foo.h'), 'bar') write_file(os.path.join(app.installdir, 'lib', 'libfoo.a'), 'libfoo') pyver = '.'.join(map(str, sys.version_info[:2])) write_file( os.path.join(app.installdir, 'lib', 'python%s' % pyver, 'site-packages', 'foo.egg'), 'foo egg') write_file( os.path.join(app.installdir, 'lib64', 'pkgconfig', 'foo.pc'), 'libfoo: foo') # PythonPackage relies on the fact that 'python' points to the right Python version tmpdir = tempfile.mkdtemp() python = os.path.join(tmpdir, 'python') write_file(python, '#!/bin/bash\necho $0 $@\n%s "$@"' % sys.executable) adjust_permissions(python, stat.S_IXUSR) os.environ['PATH'] = '%s:%s' % (tmpdir, os.getenv('PATH', '')) from easybuild.tools.filetools import which print(which('python')) # create module file app.make_module_step() remove_file(python) self.assertTrue(TMPDIR in app.installdir) self.assertTrue(TMPDIR in app.installdir_mod) modtxt = None for cand_mod_filename in ['3.14', '3.14.lua']: full_modpath = os.path.join(app.installdir_mod, 'testpypkg', cand_mod_filename) if os.path.exists(full_modpath): modtxt = read_file(full_modpath) break self.assertFalse(modtxt is None) regexs = [ (r'^prepend.path.*\WCPATH\W.*include"?\W*$', True), (r'^prepend.path.*\WLD_LIBRARY_PATH\W.*lib"?\W*$', True), (r'^prepend.path.*\WLIBRARY_PATH\W.*lib"?\W*$', True), (r'^prepend.path.*\WPATH\W.*bin"?\W*$', True), (r'^prepend.path.*\WPKG_CONFIG_PATH\W.*lib64/pkgconfig"?\W*$', True), (r'^prepend.path.*\WPYTHONPATH\W.*lib/python[23]\.[0-9]/site-packages"?\W*$', True), # lib64 doesn't contain any library files, so these are *not* included in $LD_LIBRARY_PATH or $LIBRARY_PATH (r'^prepend.path.*\WLD_LIBRARY_PATH\W.*lib64', False), (r'^prepend.path.*\WLIBRARY_PATH\W.*lib64', False), ] for (pattern, found) in regexs: regex = re.compile(pattern, re.M) if found: assert_msg = "Pattern '%s' found in: %s" % (regex.pattern, modtxt) else: assert_msg = "Pattern '%s' not found in: %s" % (regex.pattern, modtxt) self.assertEqual(bool(regex.search(modtxt)), found, assert_msg)
def template_init_test(self, easyblock, name='foo', version='1.3.2'): """Test whether all easyblocks can be initialized.""" def check_extra_options_format(extra_options): """Make sure extra_options value is of correct format.""" # EasyBuild v2.0: dict with <string> keys and <list> values self.assertTrue(isinstance(extra_options, dict)) extra_options.items() extra_options.keys() extra_options.values() for key in extra_options.keys(): self.assertTrue(isinstance(extra_options[key], list)) self.assertTrue(len(extra_options[key]), 3) class_regex = re.compile(r"^class (.*)\(.*", re.M) self.log.debug("easyblock: %s" % easyblock) # read easyblock Python module f = open(easyblock, "r") txt = f.read() f.close() regexps = [ # make sure error reporting is done correctly (no more log.error, log.exception) re.compile(r"log\.error\("), re.compile(r"log\.exception\("), re.compile(r"log\.raiseException\("), # check for use of 'basestring', which is Python 2.x only (should use string_type from tools.py2vs3 instead) re.compile(r"[^\w]basestring([^\w]|$)"), # check for use of '.iteritems()', which is Python 2.x only (should use .items instead) re.compile(r"\.iteritems\(\)"), # sys.maxint is no longer there in Python 3 re.compile(r"sys\.maxint"), ] for regexp in regexps: self.assertFalse( regexp.search(txt), "No match for '%s' in %s" % (regexp.pattern, easyblock)) # make sure that (named) arguments get passed down for prepare_step if re.search('def prepare_step', txt): regex = re.compile(r"def prepare_step\(self, \*args, \*\*kwargs\):") self.assertTrue( regex.search(txt), "Pattern '%s' found in %s" % (regex.pattern, easyblock)) if re.search(r'\.prepare_step\(', txt): regex = re.compile(r"\.prepare_step\(.*\*args,.*\*\*kwargs\.*\)") self.assertTrue( regex.search(txt), "Pattern '%s' found in %s" % (regex.pattern, easyblock)) # obtain easyblock class name using regex res = class_regex.search(txt) if res: ebname = res.group(1) self.log.debug("Found class name for easyblock %s: %s" % (easyblock, ebname)) # figure out list of mandatory variables, and define with dummy values as necessary app_class = get_easyblock_class(ebname) extra_options = app_class.extra_options() check_extra_options_format(extra_options) # extend easyconfig to make sure mandatory custom easyconfig parameters are defined extra_txt = '' for (key, val) in extra_options.items(): if val[2] == MANDATORY: # use default value if any is set, otherwise use "foo" if val[0]: test_param = val[0] else: test_param = 'foo' extra_txt += '%s = "%s"\n' % (key, test_param) # write easyconfig file self.writeEC(ebname, name=name, version=version, extratxt=extra_txt) # initialize easyblock # if this doesn't fail, the test succeeds app = app_class(EasyConfig(self.eb_file)) # check whether easyblock instance is still using functions from a deprecated location mod = __import__(app.__module__, [], [], ['easybuild.easyblocks']) moved_functions = [ 'modify_env', 'parse_log_for_error', 'read_environment', 'run_cmd', 'run_cmd_qa' ] for fn in moved_functions: if hasattr(mod, fn): tup = (fn, app.__module__, globals()[fn].__module__) self.assertTrue( getattr(mod, fn) is globals()[fn], "%s in %s is imported from %s" % tup) renamed_functions = [ ('source_paths', 'source_path'), ('get_avail_core_count', 'get_core_count'), ('get_os_type', 'get_kernel_name'), ('det_full_ec_version', 'det_installversion'), ] for (new_fn, old_fn) in renamed_functions: self.assertFalse( hasattr(mod, old_fn), "%s: %s is replaced by %s" % (app.__module__, old_fn, new_fn)) # cleanup app.close_log() os.remove(app.logfile) else: self.assertTrue(False, "Class found in easyblock %s" % easyblock)
if '.git' in subfolders: log.info("found .git subfolder, ignoring it") subfolders.remove('.git') for ec_file in files: if not ec_file.endswith('.eb') or ec_file in ["TEMPLATE.eb"]: log.warning("SKIPPING %s/%s" % (root, ec_file)) continue ec_file = join(root, ec_file) ec_file = read(ec_file) try: ec = EasyConfig(ec_file) log.info("found valid easyconfig %s" % ec) if not ec.name in names: log.info("found new software package %s" % ec) # check if an easyblock exists module = get_easyblock_class(None, name=ec.name).__module__.split('.')[-1] if module != "configuremake": ec.easyblock = module else: ec.easyblock = None configs.append(ec) names.append(ec.name) except Exception, err: log.error("faulty easyconfig %s: %s" % (ec_file, err)) log.info("Found easyconfigs: %s" % [x.name for x in configs]) # sort by name configs = sorted(configs, key=lambda config : config.name.lower()) firstl = "" # print out the configs in markdown format for the wiki
def template_module_only_test(self, easyblock, name='foo', version='1.3.2', extra_txt=''): """Test whether all easyblocks are compatible with --module-only.""" tmpdir = tempfile.mkdtemp() class_regex = re.compile("^class (.*)\(.*", re.M) self.log.debug("easyblock: %s" % easyblock) # read easyblock Python module f = open(easyblock, "r") txt = f.read() f.close() # obtain easyblock class name using regex res = class_regex.search(txt) if res: ebname = res.group(1) self.log.debug("Found class name for easyblock %s: %s" % (easyblock, ebname)) toolchain = None # figure out list of mandatory variables, and define with dummy values as necessary app_class = get_easyblock_class(ebname) # easyblocks deriving from IntelBase require a license file to be found for --module-only if app_class == IntelBase or IntelBase in app_class.__bases__: os.environ['INTEL_LICENSE_FILE'] = os.path.join( tmpdir, 'intel.lic') write_file(os.environ['INTEL_LICENSE_FILE'], '# dummy license') if app_class == EB_IMOD: # $JAVA_HOME must be set for IMOD os.environ['JAVA_HOME'] = tmpdir if app_class == EB_OpenFOAM: # proper toolchain must be used for OpenFOAM(-Extend), to determine value to set for $WM_COMPILER write_file( os.path.join(tmpdir, 'GCC', '4.9.3-2.25'), '\n'.join([ '#%Module', 'setenv EBROOTGCC %s' % tmpdir, 'setenv EBVERSIONGCC 4.9.3', ])) write_file( os.path.join(tmpdir, 'OpenMPI', '1.10.2-GCC-4.9.3-2.25'), '\n'.join([ '#%Module', 'setenv EBROOTOPENMPI %s' % tmpdir, 'setenv EBVERSIONOPENMPI 1.10.2', ])) write_file( os.path.join(tmpdir, 'gompi', '2016a'), '\n'.join([ '#%Module', 'module load GCC/4.9.3-2.25', 'module load OpenMPI/1.10.2-GCC-4.9.3-2.25', ])) os.environ['MODULEPATH'] = tmpdir toolchain = {'name': 'gompi', 'version': '2016a'} # extend easyconfig to make sure mandatory custom easyconfig paramters are defined extra_options = app_class.extra_options() for (key, val) in extra_options.items(): if val[2] == MANDATORY: extra_txt += '%s = "foo"\n' % key # write easyconfig file self.writeEC(ebname, name=name, version=version, extratxt=extra_txt, toolchain=toolchain) # initialize easyblock # if this doesn't fail, the test succeeds app = app_class(EasyConfig(self.eb_file)) # run all steps, most should be skipped orig_workdir = os.getcwd() try: app.run_all_steps(run_test_cases=False) finally: os.chdir(orig_workdir) modfile = os.path.join(TMPDIR, 'modules', 'all', name, version) luamodfile = '%s.lua' % modfile self.assertTrue( os.path.exists(modfile) or os.path.exists(luamodfile), "Module file %s or %s was generated" % (modfile, luamodfile)) if os.path.exists(modfile): modtxt = read_file(modfile) else: modtxt = read_file(luamodfile) none_regex = re.compile('None') self.assertFalse(none_regex.search(modtxt), "None not found in module file: %s" % modtxt) # cleanup app.close_log() os.remove(app.logfile) shutil.rmtree(tmpdir) else: self.assertTrue(False, "Class found in easyblock %s" % easyblock)
log.info("found .git subfolder, ignoring it") subfolders.remove('.git') for ec_file in files: if not ec_file.endswith('.eb') or ec_file in ["TEMPLATE.eb"]: log.warning("SKIPPING %s/%s" % (root, ec_file)) continue ec_file = join(root, ec_file) ec_file = read(ec_file) try: ec = EasyConfig(ec_file) log.info("found valid easyconfig %s" % ec) if not ec.name in names: log.info("found new software package %s" % ec.name) ec.easyblock = None # check if an easyblock exists ebclass = get_easyblock_class(None, name=ec.name, error_on_missing_easyblock=False) if ebclass is not None: module = ebclass.__module__.split('.')[-1] if module != "configuremake": ec.easyblock = module configs.append(ec) names.append(ec.name) except Exception, err: raise EasyBuildError("faulty easyconfig %s: %s", ec_file, err) log.info("Found easyconfigs: %s" % [x.name for x in configs]) # sort by name configs = sorted(configs, key=lambda config : config.name.lower()) firstl = "" # print out the configs in markdown format for the wiki
def template_easyconfig_test(self, spec): """Tests for an individual easyconfig: parsing, instantiating easyblock, check patches, ...""" # set to False, so it's False in case of this test failing global single_tests_ok prev_single_tests_ok = single_tests_ok single_tests_ok = False # parse easyconfig ecs = process_easyconfig(spec) if len(ecs) == 1: ec = ecs[0]['ec'] else: self.assertTrue( False, "easyconfig %s does not contain blocks, yields only one parsed easyconfig" % spec) # check easyconfig file name expected_fn = '%s-%s.eb' % (ec['name'], det_full_ec_version(ec)) msg = "Filename '%s' of parsed easyconfig matches expected filename '%s'" % ( spec, expected_fn) self.assertEqual(os.path.basename(spec), expected_fn, msg) name, easyblock = fetch_parameters_from_easyconfig(ec.rawtxt, ['name', 'easyblock']) # make sure easyconfig file is in expected location expected_subdir = os.path.join('easybuild', 'easyconfigs', name.lower()[0], name) subdir = os.path.join(*spec.split(os.path.sep)[-5:-1]) fail_msg = "Easyconfig file %s not in expected subdirectory %s" % ( spec, expected_subdir) self.assertEqual(expected_subdir, subdir, fail_msg) # sanity check for software name self.assertTrue(ec['name'], name) # instantiate easyblock with easyconfig file app_class = get_easyblock_class(easyblock, name=name) # check that automagic fallback to ConfigureMake isn't done (deprecated behaviour) fn = os.path.basename(spec) error_msg = "%s relies on automagic fallback to ConfigureMake, should use easyblock = 'ConfigureMake' instead" % fn self.assertTrue(easyblock or not app_class is ConfigureMake, error_msg) app = app_class(ec) # more sanity checks self.assertTrue(name, app.name) self.assertTrue(ec['version'], app.version) # make sure all patch files are available specdir = os.path.dirname(spec) specfn = os.path.basename(spec) for patch in ec['patches']: if isinstance(patch, (tuple, list)): patch = patch[0] # only check actual patch files, not other files being copied via the patch functionality if patch.endswith('.patch'): patch_full = os.path.join(specdir, patch) msg = "Patch file %s is available for %s" % (patch_full, specfn) self.assertTrue(os.path.isfile(patch_full), msg) ext_patches = [] for ext in ec['exts_list']: if isinstance(ext, (tuple, list)) and len(ext) == 3: self.assertTrue(isinstance(ext[2], dict), "3rd element of extension spec is a dictionary") for ext_patch in ext[2].get('patches', []): if isinstance(ext_patch, (tuple, list)): ext_patch = ext_patch[0] # only check actual patch files, not other files being copied via the patch functionality if ext_patch.endswith('.patch'): ext_patch_full = os.path.join(specdir, ext_patch) msg = "Patch file %s is available for %s" % ( ext_patch_full, specfn) self.assertTrue(os.path.isfile(ext_patch_full), msg) # check whether all extra_options defined for used easyblock are defined for key in app.extra_options(): self.assertTrue(key in app.cfg) app.close_log() os.remove(app.logfile) # dump the easyconfig file handle, test_ecfile = tempfile.mkstemp() os.close(handle) ec.dump(test_ecfile) dumped_ec = EasyConfig(test_ecfile) os.remove(test_ecfile) # inject dummy values for templates that are only known at a later stage dummy_template_values = { 'builddir': '/dummy/builddir', 'installdir': '/dummy/installdir', } ec.template_values.update(dummy_template_values) dumped_ec.template_values.update(dummy_template_values) for key in sorted(ec._config): self.assertEqual(ec[key], dumped_ec[key]) # cache the parsed easyconfig, to avoid that it is parsed again self.parsed_easyconfigs.append(ecs[0]) # test passed, so set back to True single_tests_ok = True and prev_single_tests_ok
log.info("found .git subfolder, ignoring it") subfolders.remove('.git') for ec_file in files: if not ec_file.endswith('.eb') or ec_file in ["TEMPLATE.eb"]: log.warning("SKIPPING %s/%s" % (root, ec_file)) continue ec_file = join(root, ec_file) ec_file = read(ec_file) try: ec = EasyConfig(ec_file) log.info("found valid easyconfig %s" % ec) if not ec.name in names: log.info("found new software package %s" % ec) ec.easyblock = None # check if an easyblock exists ebclass = get_easyblock_class(None, name=ec.name, default_fallback=False) if ebclass is not None: module = ebclass.__module__.split('.')[-1] if module != "configuremake": ec.easyblock = module configs.append(ec) names.append(ec.name) except Exception, err: log.error("faulty easyconfig %s: %s" % (ec_file, err)) log.info("Found easyconfigs: %s" % [x.name for x in configs]) # sort by name configs = sorted(configs, key=lambda config : config.name.lower()) firstl = "" # print out the configs in markdown format for the wiki
def template_module_only_test(self, easyblock, name='foo', version='1.3.2', extra_txt=''): """Test whether all easyblocks are compatible with --module-only.""" tmpdir = tempfile.mkdtemp() class_regex = re.compile("^class (.*)\(.*", re.M) self.log.debug("easyblock: %s" % easyblock) # read easyblock Python module f = open(easyblock, "r") txt = f.read() f.close() # obtain easyblock class name using regex res = class_regex.search(txt) if res: ebname = res.group(1) self.log.debug("Found class name for easyblock %s: %s" % (easyblock, ebname)) toolchain = None # figure out list of mandatory variables, and define with dummy values as necessary app_class = get_easyblock_class(ebname) # easyblocks deriving from IntelBase require a license file to be found for --module-only bases = list(app_class.__bases__) for base in copy.copy(bases): bases.extend(base.__bases__) if app_class == IntelBase or IntelBase in bases: os.environ['INTEL_LICENSE_FILE'] = os.path.join( tmpdir, 'intel.lic') write_file(os.environ['INTEL_LICENSE_FILE'], '# dummy license') elif app_class == EB_IMOD: # $JAVA_HOME must be set for IMOD os.environ['JAVA_HOME'] = tmpdir elif app_class == PythonBundle: # $EBROOTPYTHON must be set for PythonBundle easyblock os.environ[ 'EBROOTPYTHON'] = '/fake/install/prefix/Python/2.7.14-foss-2018a' elif app_class == EB_OpenFOAM: # proper toolchain must be used for OpenFOAM(-Extend), to determine value to set for $WM_COMPILER write_file( os.path.join(tmpdir, 'GCC', '4.9.3-2.25'), '\n'.join([ '#%Module', 'setenv EBROOTGCC %s' % tmpdir, 'setenv EBVERSIONGCC 4.9.3', ])) write_file( os.path.join(tmpdir, 'OpenMPI', '1.10.2-GCC-4.9.3-2.25'), '\n'.join([ '#%Module', 'setenv EBROOTOPENMPI %s' % tmpdir, 'setenv EBVERSIONOPENMPI 1.10.2', ])) write_file( os.path.join(tmpdir, 'gompi', '2016a'), '\n'.join([ '#%Module', 'module load GCC/4.9.3-2.25', 'module load OpenMPI/1.10.2-GCC-4.9.3-2.25', ])) os.environ['MODULEPATH'] = tmpdir toolchain = {'name': 'gompi', 'version': '2016a'} # extend easyconfig to make sure mandatory custom easyconfig paramters are defined extra_options = app_class.extra_options() for (key, val) in extra_options.items(): if val[2] == MANDATORY: extra_txt += '%s = "foo"\n' % key # write easyconfig file self.writeEC(ebname, name=name, version=version, extratxt=extra_txt, toolchain=toolchain) # take into account that for some easyblock, particular dependencies are hard required early on # (in prepare_step for exampel); # we just set the corresponding $EBROOT* environment variables here to fool it... req_deps = { # QScintilla easyblock requires that either PyQt or PyQt5 are available as dependency # (PyQt is easier, since PyQt5 is only supported for sufficiently recent QScintilla versions) 'qscintilla.py': [('PyQt', '4.12')], # MotionCor2 and Gctf easyblock requires CUDA as dependency 'motioncor2.py': [('CUDA', '10.1.105')], 'gctf.py': [('CUDA', '10.1.105')], } easyblock_fn = os.path.basename(easyblock) for (dep_name, dep_version) in req_deps.get(easyblock_fn, []): dep_root_envvar = get_software_root_env_var_name(dep_name) os.environ[dep_root_envvar] = '/value/should/not/matter' dep_version_envvar = get_software_version_env_var_name(dep_name) os.environ[dep_version_envvar] = dep_version # initialize easyblock # if this doesn't fail, the test succeeds app = app_class(EasyConfig(self.eb_file)) # run all steps, most should be skipped orig_workdir = os.getcwd() try: app.run_all_steps(run_test_cases=False) finally: change_dir(orig_workdir) if os.path.basename(easyblock) == 'modulerc.py': # .modulerc must be cleaned up to avoid causing trouble (e.g. "Duplicate version symbol" errors) modulerc = os.path.join(TMPDIR, 'modules', 'all', name, '.modulerc') if os.path.exists(modulerc): remove_file(modulerc) modulerc += '.lua' if os.path.exists(modulerc): remove_file(modulerc) else: modfile = os.path.join(TMPDIR, 'modules', 'all', name, version) luamodfile = '%s.lua' % modfile self.assertTrue( os.path.exists(modfile) or os.path.exists(luamodfile), "Module file %s or %s was generated" % (modfile, luamodfile)) if os.path.exists(modfile): modtxt = read_file(modfile) else: modtxt = read_file(luamodfile) none_regex = re.compile('None') self.assertFalse(none_regex.search(modtxt), "None not found in module file: %s" % modtxt) # cleanup app.close_log() remove_file(app.logfile) remove_dir(tmpdir) else: self.assertTrue(False, "Class found in easyblock %s" % easyblock)
def template_easyconfig_test(self, spec): """Tests for an individual easyconfig: parsing, instantiating easyblock, check patches, ...""" # set to False, so it's False in case of this test failing global single_tests_ok prev_single_tests_ok = single_tests_ok single_tests_ok = False # parse easyconfig ecs = process_easyconfig(spec) if len(ecs) == 1: ec = ecs[0]['ec'] else: self.assertTrue( False, "easyconfig %s does not contain blocks, yields only one parsed easyconfig" % spec) # check easyconfig file name expected_fn = '%s-%s.eb' % (ec['name'], det_full_ec_version(ec)) msg = "Filename '%s' of parsed easconfig matches expected filename '%s'" % ( spec, expected_fn) self.assertEqual(os.path.basename(spec), expected_fn, msg) # sanity check for software name name = fetch_parameter_from_easyconfig_file(spec, 'name') self.assertTrue(ec['name'], name) # try and fetch easyblock spec from easyconfig easyblock = fetch_parameter_from_easyconfig_file(spec, 'easyblock') # instantiate easyblock with easyconfig file app_class = get_easyblock_class(easyblock, name=name) # check that automagic fallback to ConfigureMake isn't done (deprecated behaviour) fn = os.path.basename(spec) error_msg = "%s relies on automagic fallback to ConfigureMake, should use easyblock = 'ConfigureMake' instead" % fn self.assertTrue(easyblock or not app_class is ConfigureMake, error_msg) app = app_class(ec) # more sanity checks self.assertTrue(name, app.name) self.assertTrue(ec['version'], app.version) # make sure all patch files are available specdir = os.path.dirname(spec) specfn = os.path.basename(spec) for patch in ec['patches']: if isinstance(patch, (tuple, list)): patch = patch[0] # only check actual patch files, not other files being copied via the patch functionality if patch.endswith('.patch'): patch_full = os.path.join(specdir, patch) msg = "Patch file %s is available for %s" % (patch_full, specfn) self.assertTrue(os.path.isfile(patch_full), msg) ext_patches = [] for ext in ec['exts_list']: if isinstance(ext, (tuple, list)) and len(ext) == 3: self.assertTrue(isinstance(ext[2], dict), "3rd element of extension spec is a dictionary") for ext_patch in ext[2].get('patches', []): if isinstance(ext_patch, (tuple, list)): ext_patch = ext_patch[0] # only check actual patch files, not other files being copied via the patch functionality if ext_patch.endswith('.patch'): ext_patch_full = os.path.join(specdir, ext_patch) msg = "Patch file %s is available for %s" % ( ext_patch_full, specfn) self.assertTrue(os.path.isfile(ext_patch_full), msg) app.close_log() os.remove(app.logfile) # cache the parsed easyconfig, to avoid that it is parsed again self.parsed_easyconfigs.append(ecs[0]) # test passed, so set back to True single_tests_ok = True and prev_single_tests_ok