def test_validation(self): """ test other validations beside mandatory variables """ self.contents = '\n'.join([ 'name = "pi"', 'version = "3.14"', 'homepage = "http://google.com"', 'description = "test easyconfig"', 'toolchain = {"name":"dummy", "version": "dummy"}', 'stop = "notvalid"', ]) self.prep() eb = EasyConfig(self.eb_file, validate=False, valid_stops=self.all_stops) self.assertErrorRegex(EasyBuildError, r"\w* provided '\w*' is not valid", eb.validate) eb['stop'] = 'patch' # this should now not crash eb.validate() eb['osdependencies'] = ['non-existent-dep'] self.assertErrorRegex(EasyBuildError, "OS dependencies were not found", eb.validate) # dummy toolchain, installversion == version self.assertEqual(eb.get_installversion(), "3.14") os.chmod(self.eb_file, 0000) self.assertErrorRegex(EasyBuildError, "Unexpected IOError", EasyConfig, self.eb_file) os.chmod(self.eb_file, 0755) self.contents += "\nsyntax_error'" self.prep() self.assertErrorRegex(EasyBuildError, "SyntaxError", EasyConfig, self.eb_file)
def test_parse_yeb(self): """Test parsing of .yeb easyconfigs.""" if 'yaml' not in sys.modules: print "Skipping test_parse_yeb (no PyYAML available)" return testdir = os.path.dirname(os.path.abspath(__file__)) test_easyconfigs = os.path.join(testdir, 'easyconfigs') test_yeb_easyconfigs = os.path.join(testdir, 'easyconfigs', 'yeb') # test parsing test_files = [ 'bzip2-1.0.6-GCC-4.9.2', 'gzip-1.6-GCC-4.9.2', 'goolf-1.4.10', 'ictce-4.1.13', 'SQLite-3.8.10.2-goolf-1.4.10', 'Python-2.7.10-ictce-4.1.13', 'CrayCCE-5.1.29', ] for filename in test_files: ec_yeb = EasyConfig(os.path.join(test_yeb_easyconfigs, '%s.yeb' % filename)) # compare with parsed result of .eb easyconfig ec_eb = EasyConfig(os.path.join(test_easyconfigs, '%s.eb' % filename)) no_match = False for key in sorted(ec_yeb.asdict()): eb_val = ec_eb[key] yeb_val = ec_yeb[key] if key == 'description': # multi-line string is always terminated with '\n' in YAML, so strip it off yeb_val = yeb_val.strip() self.assertEqual(yeb_val, eb_val)
def test_templating(self): """ test easyconfig templating """ inp = { 'name':'PI', 'version':'3.14', 'namelower':'pi', 'cmd': 'tar xfvz %s', } # don't use any escaping insanity here, since it is templated itself self.contents = '\n'.join([ 'name = "%(name)s"', 'version = "%(version)s"', 'homepage = "http://google.com"', 'description = "test easyconfig %%(name)s"', 'toolchain = {"name":"dummy", "version": "dummy2"}', 'source_urls = [(GOOGLECODE_SOURCE)]', 'sources = [SOURCE_TAR_GZ, (SOURCELOWER_TAR_GZ, "%(cmd)s")]', 'sanity_check_paths = {"files": [], "dirs": ["libfoo.%%s" %% SHLIB_EXT]}', ]) % inp self.prep() eb = EasyConfig(self.eb_file, validate=False, valid_stops=self.all_stops) eb.validate() eb.generate_template_values() self.assertEqual(eb['description'], "test easyconfig PI") const_dict = dict([(x[0], x[1]) for x in easyconfig.templates.TEMPLATE_CONSTANTS]) self.assertEqual(eb['sources'][0], const_dict['SOURCE_TAR_GZ'] % inp) self.assertEqual(eb['sources'][1][0], const_dict['SOURCELOWER_TAR_GZ'] % inp) self.assertEqual(eb['sources'][1][1], 'tar xfvz %s') self.assertEqual(eb['source_urls'][0], const_dict['GOOGLECODE_SOURCE'] % inp) self.assertEqual(eb['sanity_check_paths']['dirs'][0], 'libfoo.%s' % get_shared_lib_ext()) # test the escaping insanity here (ie all the crap we allow in easyconfigs) eb['description'] = "test easyconfig % %% %s% %%% %(name)s %%(name)s %%%(name)s %%%%(name)s" self.assertEqual(eb['description'], "test easyconfig % %% %s% %%% PI %(name)s %PI %%(name)s")
def test_validation(self): """ test other validations beside mandatory parameters """ self.contents = '\n'.join([ 'easyblock = "ConfigureMake"', 'name = "pi"', 'version = "3.14"', 'homepage = "http://example.com"', 'description = "test easyconfig"', 'toolchain = {"name":"dummy", "version": "dummy"}', 'stop = "notvalid"', ]) self.prep() ec = EasyConfig(self.eb_file, validate=False) self.assertErrorRegex(EasyBuildError, r"\w* provided '\w*' is not valid", ec.validate) ec['stop'] = 'patch' # this should now not crash ec.validate() ec['osdependencies'] = ['non-existent-dep'] self.assertErrorRegex(EasyBuildError, "OS dependencies were not found", ec.validate) # dummy toolchain, installversion == version self.assertEqual(det_full_ec_version(ec), "3.14") os.chmod(self.eb_file, 0000) self.assertErrorRegex(EasyBuildError, "Permission denied", EasyConfig, self.eb_file) os.chmod(self.eb_file, 0755) self.contents += "\nsyntax_error'" self.prep() self.assertErrorRegex(EasyBuildError, "SyntaxError", EasyConfig, self.eb_file)
def test_filter_deps(self): """Test filtered dependencies.""" test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs') ec_file = os.path.join(test_ecs_dir, 'goolf-1.4.10.eb') ec = EasyConfig(ec_file) deps = sorted([dep['name'] for dep in ec.dependencies()]) self.assertEqual(deps, ['FFTW', 'GCC', 'OpenBLAS', 'OpenMPI', 'ScaLAPACK']) # test filtering multiple deps init_config(build_options={'filter_deps': ['FFTW', 'ScaLAPACK']}) deps = sorted([dep['name'] for dep in ec.dependencies()]) self.assertEqual(deps, ['GCC', 'OpenBLAS', 'OpenMPI']) # test filtering of non-existing dep init_config(build_options={'filter_deps': ['zlib']}) deps = sorted([dep['name'] for dep in ec.dependencies()]) self.assertEqual(deps, ['FFTW', 'GCC', 'OpenBLAS', 'OpenMPI', 'ScaLAPACK']) # test parsing of value passed to --filter-deps opts = init_config(args=[]) self.assertEqual(opts.filter_deps, None) opts = init_config(args=['--filter-deps=zlib']) self.assertEqual(opts.filter_deps, ['zlib']) opts = init_config(args=['--filter-deps=zlib,ncurses']) self.assertEqual(opts.filter_deps, ['zlib', 'ncurses'])
def test_validation(self): """ test other validations beside mandatory variables """ self.contents = '\n'.join([ 'name = "pi"', 'version = "3.14"', 'homepage = "http://example.com"', 'description = "test easyconfig"', 'toolchain = {"name":"dummy", "version": "dummy"}', 'stop = "notvalid"', ]) self.prep() ec = EasyConfig(self.eb_file, validate=False) self.assertErrorRegex(EasyBuildError, r"\w* provided '\w*' is not valid", ec.validate) ec['stop'] = 'patch' # this should now not crash ec.validate() ec['osdependencies'] = ['non-existent-dep'] self.assertErrorRegex(EasyBuildError, "OS dependencies were not found", ec.validate) # dummy toolchain, installversion == version self.assertEqual(det_full_ec_version(ec), "3.14") os.chmod(self.eb_file, 0000) self.assertErrorRegex(EasyBuildError, "Permission denied", EasyConfig, self.eb_file) os.chmod(self.eb_file, 0755) self.contents += "\nsyntax_error'" self.prep() self.assertErrorRegex(EasyBuildError, "SyntaxError", EasyConfig, self.eb_file)
def test_extensions_step(self): """Test the extensions_step""" self.contents = '\n'.join([ 'easyblock = "ConfigureMake"', 'name = "pi"', 'version = "3.14"', 'homepage = "http://example.com"', 'description = "test easyconfig"', 'toolchain = {"name": "dummy", "version": "dummy"}', 'exts_list = ["ext1"]', ]) self.writeEC() """Testcase for extensions""" # test for proper error message without the exts_defaultclass set eb = EasyBlock(EasyConfig(self.eb_file)) eb.installdir = config.install_path() self.assertRaises(EasyBuildError, eb.extensions_step, fetch=True) self.assertErrorRegex(EasyBuildError, "No default extension class set", eb.extensions_step, fetch=True) # test if everything works fine if set self.contents += "\nexts_defaultclass = 'DummyExtension'" self.writeEC() eb = EasyBlock(EasyConfig(self.eb_file)) eb.builddir = config.build_path() eb.installdir = config.install_path() eb.extensions_step(fetch=True) # test for proper error message when skip is set, but no exts_filter is set self.assertRaises(EasyBuildError, eb.skip_extensions) self.assertErrorRegex(EasyBuildError, "no exts_filter set", eb.skip_extensions) # cleanup eb.close_log() os.remove(eb.logfile)
def test_format_equivalence_basic(self): """Test whether easyconfigs in different formats are equivalent.""" # hard enable experimental orig_experimental = easybuild.tools.build_log.EXPERIMENTAL easybuild.tools.build_log.EXPERIMENTAL = True easyconfigs_path = os.path.join(os.path.dirname(__file__), 'easyconfigs') # set max diff high enough to make sure the difference is shown in case of problems self.maxDiff = 10000 for eb_file1, eb_file2, specs in [ ('gzip-1.4.eb', 'gzip.eb', {}), ('gzip-1.4.eb', 'gzip.eb', {'version': '1.4'}), ('gzip-1.4.eb', 'gzip.eb', {'version': '1.4', 'toolchain': {'name': 'dummy', 'version': 'dummy'}}), ('gzip-1.4-GCC-4.6.3.eb', 'gzip.eb', {'version': '1.4', 'toolchain': {'name': 'GCC', 'version': '4.6.3'}}), ('gzip-1.5-goolf-1.4.10.eb', 'gzip.eb', {'version': '1.5', 'toolchain': {'name': 'goolf', 'version': '1.4.10'}}), ('gzip-1.5-ictce-4.1.13.eb', 'gzip.eb', {'version': '1.5', 'toolchain': {'name': 'ictce', 'version': '4.1.13'}}), ]: ec1 = EasyConfig(os.path.join(easyconfigs_path, 'v1.0', eb_file1), validate=False) ec2 = EasyConfig(os.path.join(easyconfigs_path, 'v2.0', eb_file2), validate=False, build_specs=specs) ec2_dict = ec2.asdict() # reset mandatory attributes from format2 that are not in format 1 for attr in ['docurls', 'software_license', 'software_license_urls']: ec2_dict[attr] = None self.assertEqual(ec1.asdict(), ec2_dict) # restore easybuild.tools.build_log.EXPERIMENTAL = orig_experimental
def test_format_equivalence_basic(self): """Test whether easyconfigs in different formats are equivalent.""" # hard enable experimental orig_experimental = easybuild.tools.build_log.EXPERIMENTAL easybuild.tools.build_log.EXPERIMENTAL = True easyconfigs_path = os.path.join(os.path.dirname(__file__), 'easyconfigs') # set max diff high enough to make sure the difference is shown in case of problems self.maxDiff = 10000 build_options = { 'valid_stops': self.all_stops, 'validate': False, } for eb_file1, eb_file2, specs in [ ('gzip-1.4.eb', 'gzip.eb', {}), ('gzip-1.4.eb', 'gzip.eb', {'version': '1.4'}), ('gzip-1.4.eb', 'gzip.eb', {'version': '1.4', 'toolchain': {'name': 'dummy', 'version': 'dummy'}}), ('gzip-1.4-GCC-4.6.3.eb', 'gzip.eb', {'version': '1.4', 'toolchain': {'name': 'GCC', 'version': '4.6.3'}}), ('gzip-1.5-goolf-1.4.10.eb', 'gzip.eb', {'version': '1.5', 'toolchain': {'name': 'goolf', 'version': '1.4.10'}}), ('gzip-1.5-ictce-4.1.13.eb', 'gzip.eb', {'version': '1.5', 'toolchain': {'name': 'ictce', 'version': '4.1.13'}}), ]: ec1 = EasyConfig(os.path.join(easyconfigs_path, 'v1.0', eb_file1), build_options=build_options) ec2 = EasyConfig(os.path.join(easyconfigs_path, 'v2.0', eb_file2), build_options=build_options, build_specs=specs) self.assertEqual(ec1.asdict(), ec2.asdict()) # restore easybuild.tools.build_log.EXPERIMENTAL = orig_experimental
def test_external_module_toolchain(self): """Test specifying external (build) dependencies in yaml format.""" if 'yaml' not in sys.modules: print "Skipping test_external_module_toolchain (no PyYAML available)" return ecpath = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'yeb', 'CrayCCE-5.1.29.yeb') metadata = { 'name': ['foo', 'bar'], 'version': ['1.2.3', '3.2.1'], 'prefix': '/foo/bar', } build_options = { 'external_modules_metadata': { 'fftw/3.3.4.0': metadata }, 'valid_module_classes': module_classes(), } init_config(build_options=build_options) easybuild.tools.build_log.EXPERIMENTAL = True ec = EasyConfig(ecpath) self.assertEqual(ec.dependencies()[1]['full_mod_name'], 'fftw/3.3.4.0') self.assertEqual(ec.dependencies()[1]['external_module_metadata'], metadata)
def test_check_readiness(self): """Test check_readiness method.""" init_config(build_options={'validate': False}) # check that check_readiness step works (adding dependencies, etc.) ec_file = 'OpenMPI-1.6.4-GCC-4.6.4.eb' ec_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', ec_file) ec = EasyConfig(ec_path) eb = EasyBlock(ec) eb.check_readiness_step() # a proper error should be thrown for dependencies that can't be resolved (module should be there) tmpdir = tempfile.mkdtemp() shutil.copy2(ec_path, tmpdir) ec_path = os.path.join(tmpdir, ec_file) f = open(ec_path, 'a') f.write("\ndependencies += [('nosuchsoftware', '1.2.3')]\n") f.close() ec = EasyConfig(ec_path) eb = EasyBlock(ec) try: eb.check_readiness_step() except EasyBuildError, err: err_regex = re.compile( "Missing modules for one or more dependencies: nosuchsoftware/1.2.3-GCC-4.6.4" ) self.assertTrue( err_regex.search(str(err)), "Pattern '%s' found in '%s'" % (err_regex.pattern, err))
def process_easyconfig(path, build_options=None, build_specs=None): """ Process easyconfig, returning some information for each block @param path: path to easyconfig file @param build_options: dictionary specifying build options (e.g. robot_path, check_osdeps, ...) @param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...) """ blocks = retrieve_blocks_in_spec(path, build_options.get("only_blocks", None)) easyconfigs = [] for spec in blocks: # process for dependencies and real installversionname _log.debug("Processing easyconfig %s" % spec) # create easyconfig try: ec = EasyConfig(spec, build_options=build_options, build_specs=build_specs) except EasyBuildError, err: msg = "Failed to process easyconfig %s:\n%s" % (spec, err.msg) _log.exception(msg) name = ec["name"] # this app will appear as following module in the list easyconfig = { "ec": ec, "spec": spec, "module": det_full_module_name(ec), "dependencies": [], "builddependencies": [], } if len(blocks) > 1: easyconfig["originalSpec"] = path # add build dependencies for dep in ec.builddependencies(): _log.debug("Adding build dependency %s for app %s." % (dep, name)) easyconfig["builddependencies"].append(dep) # add dependencies (including build dependencies) for dep in ec.dependencies(): _log.debug("Adding dependency %s for app %s." % (dep, name)) easyconfig["dependencies"].append(dep) # add toolchain as dependency too if ec.toolchain.name != DUMMY_TOOLCHAIN_NAME: dep = ec.toolchain.as_dict() _log.debug("Adding toolchain %s as dependency for app %s." % (dep, name)) easyconfig["dependencies"].append(dep) del ec # this is used by the parallel builder easyconfig["unresolved_deps"] = copy.deepcopy(easyconfig["dependencies"]) easyconfigs.append(easyconfig)
def process_easyconfig(path, onlyBlocks=None, regtest_online=False, validate=True): """ Process easyconfig, returning some information for each block """ blocks = retrieve_blocks_in_spec(path, onlyBlocks) easyconfigs = [] for spec in blocks: # process for dependencies and real installversionname # - use mod? __init__ and importCfg are ignored. _log.debug("Processing easyconfig %s" % spec) # create easyconfig try: all_stops = [x[0] for x in EasyBlock.get_steps()] ec = EasyConfig(spec, validate=validate, valid_module_classes=module_classes(), valid_stops=all_stops) except EasyBuildError, err: msg = "Failed to process easyconfig %s:\n%s" % (spec, err.msg) _log.exception(msg) name = ec['name'] # this app will appear as following module in the list easyconfig = { 'spec': spec, 'module': (ec.name, ec.get_installversion()), 'dependencies': [], 'builddependencies': [], } if len(blocks) > 1: easyconfig['originalSpec'] = path # add build dependencies for dep in ec.builddependencies(): deptup = (dep['name'], dep['tc']) _log.debug("Adding build dependency %s for app %s." % (deptup, name)) easyconfig['builddependencies'].append(deptup) # add dependencies (including build dependencies) for dep in ec.dependencies(): deptup = (dep['name'], dep['tc']) _log.debug("Adding dependency %s for app %s." % (deptup, name)) easyconfig['dependencies'].append(deptup) # add toolchain as dependency too if ec.toolchain.name != 'dummy': dep = (ec.toolchain.name, ec.toolchain.version) _log.debug("Adding toolchain %s as dependency for app %s." % (dep, name)) easyconfig['dependencies'].append(dep) del ec # this is used by the parallel builder easyconfig['unresolvedDependencies'] = copy.copy(easyconfig['dependencies']) easyconfigs.append(easyconfig)
def process_easyconfig(path, onlyBlocks=None, regtest_online=False, validate=True): """ Process easyconfig, returning some information for each block """ blocks = retrieve_blocks_in_spec(path, onlyBlocks) easyconfigs = [] for spec in blocks: # process for dependencies and real installversionname # - use mod? __init__ and importCfg are ignored. _log.debug("Processing easyconfig %s" % spec) # create easyconfig try: all_stops = [x[0] for x in EasyBlock.get_steps()] ec = EasyConfig(spec, validate=validate, valid_module_classes=module_classes(), valid_stops=all_stops) except EasyBuildError, err: msg = "Failed to process easyconfig %s:\n%s" % (spec, err.msg) _log.exception(msg) name = ec["name"] # this app will appear as following module in the list easyconfig = { "spec": spec, "module": (ec.name, ec.get_installversion()), "dependencies": [], "builddependencies": [], } if len(blocks) > 1: easyconfig["originalSpec"] = path # add build dependencies for dep in ec.builddependencies(): deptup = (dep["name"], dep["tc"]) _log.debug("Adding build dependency %s for app %s." % (deptup, name)) easyconfig["builddependencies"].append(deptup) # add dependencies (including build dependencies) for dep in ec.dependencies(): deptup = (dep["name"], dep["tc"]) _log.debug("Adding dependency %s for app %s." % (deptup, name)) easyconfig["dependencies"].append(deptup) # add toolchain as dependency too if ec.toolchain.name != "dummy": dep = (ec.toolchain.name, ec.toolchain.version) _log.debug("Adding toolchain %s as dependency for app %s." % (dep, name)) easyconfig["dependencies"].append(dep) del ec # this is used by the parallel builder easyconfig["unresolvedDependencies"] = copy.copy(easyconfig["dependencies"]) easyconfigs.append(easyconfig)
def process_easyconfig(path, onlyBlocks=None, regtest_online=False, validate=True): """ Process easyconfig, returning some information for each block """ blocks = retrieve_blocks_in_spec(path, onlyBlocks) easyconfigs = [] for spec in blocks: # process for dependencies and real installversionname # - use mod? __init__ and importCfg are ignored. _log.debug("Processing easyconfig %s" % spec) # create easyconfig try: all_stops = [x[0] for x in EasyBlock.get_steps()] ec = EasyConfig(spec, validate=validate, valid_module_classes=module_classes(), valid_stops=all_stops) except EasyBuildError, err: msg = "Failed to process easyconfig %s:\n%s" % (spec, err.msg) _log.exception(msg) name = ec['name'] # this app will appear as following module in the list easyconfig = { 'ec': ec, 'spec': spec, 'module': det_full_module_name(ec), 'dependencies': [], 'builddependencies': [], } if len(blocks) > 1: easyconfig['originalSpec'] = path # add build dependencies for dep in ec.builddependencies(): _log.debug("Adding build dependency %s for app %s." % (dep, name)) easyconfig['builddependencies'].append(dep) # add dependencies (including build dependencies) for dep in ec.dependencies(): _log.debug("Adding dependency %s for app %s." % (dep, name)) easyconfig['dependencies'].append(dep) # add toolchain as dependency too if ec.toolchain.name != DUMMY_TOOLCHAIN_NAME: dep = ec.toolchain.as_dict() _log.debug("Adding toolchain %s as dependency for app %s." % (dep, name)) easyconfig['dependencies'].append(dep) del ec # this is used by the parallel builder easyconfig['unresolved_deps'] = copy.deepcopy(easyconfig['dependencies']) easyconfigs.append(easyconfig)
def test_extra_options(self): """ extra_options should allow other variables to be stored """ self.contents = '\n'.join([ 'name = "pi"', 'version = "3.14"', 'homepage = "http://google.com"', 'description = "test easyconfig"', 'toolchain = {"name":"GCC", "version": "4.6.3"}', 'toolchainopts = { "static": True}', 'dependencies = [("first", "1.1"), {"name": "second", "version": "2.2"}]', ]) self.prep() eb = EasyConfig(self.eb_file, valid_stops=self.all_stops) self.assertRaises(KeyError, lambda: eb['custom_key']) extra_vars = [('custom_key', ['default', "This is a default key", easyconfig.CUSTOM])] eb = EasyConfig(self.eb_file, extra_vars, valid_stops=self.all_stops) self.assertEqual(eb['custom_key'], 'default') eb['custom_key'] = "not so default" self.assertEqual(eb['custom_key'], 'not so default') self.contents += "\ncustom_key = 'test'" self.prep() eb = EasyConfig(self.eb_file, extra_vars, valid_stops=self.all_stops) self.assertEqual(eb['custom_key'], 'test') eb['custom_key'] = "not so default" self.assertEqual(eb['custom_key'], 'not so default') # test if extra toolchain options are being passed self.assertEqual(eb.toolchain.options['static'], True) extra_vars.extend([ ('mandatory_key', ['default', 'another mandatory key', easyconfig.MANDATORY]) ]) # test extra mandatory vars self.assertErrorRegex(EasyBuildError, r"mandatory variables \S* not provided", EasyConfig, self.eb_file, extra_vars) self.contents += '\nmandatory_key = "value"' self.prep() eb = EasyConfig(self.eb_file, extra_vars, valid_stops=self.all_stops) self.assertEqual(eb['mandatory_key'], 'value')
def test_make_module_extend_modpath(self): """Test for make_module_extend_modpath""" self.contents = '\n'.join([ 'easyblock = "ConfigureMake"', 'name = "pi"', 'version = "3.14"', 'homepage = "http://example.com"', 'description = "test easyconfig"', 'toolchain = {"name":"dummy", "version": "dummy"}', 'moduleclass = "compiler"', ]) self.writeEC() eb = EasyBlock(EasyConfig(self.eb_file)) eb.installdir = config.install_path() # no $MODULEPATH extensions for default module naming scheme (EasyBuildMNS) self.assertEqual(eb.make_module_extend_modpath(), '') usermodsdir = 'my/own/modules' modclasses = ['compiler', 'tools'] os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'CategorizedHMNS' build_options = { 'subdir_user_modules': usermodsdir, 'valid_module_classes': modclasses, } init_config(build_options=build_options) eb = EasyBlock(EasyConfig(self.eb_file)) eb.installdir = config.install_path() txt = eb.make_module_extend_modpath() if get_module_syntax() == 'Tcl': regexs = [r'^module use ".*/modules/all/Compiler/pi/3.14/%s"$' % c for c in modclasses] home = r'\$env\(HOME\)' regexs.extend([ # extension for user modules is guarded r'if { \[ file isdirectory \[ file join %s "%s/Compiler/pi/3.14" \] \] } {$' % (home, usermodsdir), # no per-moduleclass extension for user modules r'^\s+module use \[ file join %s "%s/Compiler/pi/3.14"\ ]$' % (home, usermodsdir), ]) elif get_module_syntax() == 'Lua': regexs = [r'^prepend_path\("MODULEPATH", ".*/modules/all/Compiler/pi/3.14/%s"\)$' % c for c in modclasses] home = r'os.getenv\("HOME"\)' regexs.extend([ # extension for user modules is guarded r'if isDir\(pathJoin\(%s, "%s/Compiler/pi/3.14"\)\) then' % (home, usermodsdir), # no per-moduleclass extension for user modules r'\s+prepend_path\("MODULEPATH", pathJoin\(%s, "%s/Compiler/pi/3.14"\)\)' % (home, usermodsdir), ]) else: self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) for regex in regexs: regex = re.compile(regex, re.M) self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt))
def test_extensions_sanity_check(self): """Test sanity check aspect of extensions.""" test_ecs_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'test_ecs') toy_ec = EasyConfig(os.path.join(test_ecs_dir, 't', 'toy', 'toy-0.0-gompi-1.3.12-test.eb')) # purposely put sanity check command in place that breaks the build, # to check whether sanity check is only run once; # sanity check commands are checked after checking sanity check paths, so this should work toy_ec.update('sanity_check_commands', [("%(installdir)s/bin/toy && rm %(installdir)s/bin/toy", '')]) # this import only works here, since EB_toy is a test easyblock from easybuild.easyblocks.toy import EB_toy eb = EB_toy(toy_ec) eb.silent = True eb.run_all_steps(True)
def test_extra_options(self): """ extra_options should allow other variables to be stored """ self.contents = '\n'.join([ 'easyblock = "ConfigureMake"', 'name = "pi"', 'version = "3.14"', 'homepage = "http://example.com"', 'description = "test easyconfig"', 'toolchain = {"name":"GCC", "version": "4.6.3"}', 'toolchainopts = { "static": True}', 'dependencies = [("first", "1.1"), {"name": "second", "version": "2.2"}]', ]) self.prep() eb = EasyConfig(self.eb_file) self.assertErrorRegex(EasyBuildError, "unknown easyconfig parameter", lambda: eb['custom_key']) extra_vars = {'custom_key': ['default', "This is a default key", easyconfig.CUSTOM]} eb = EasyConfig(self.eb_file, extra_options=extra_vars) self.assertEqual(eb['custom_key'], 'default') eb['custom_key'] = "not so default" self.assertEqual(eb['custom_key'], 'not so default') self.contents += "\ncustom_key = 'test'" self.prep() eb = EasyConfig(self.eb_file, extra_options=extra_vars) self.assertEqual(eb['custom_key'], 'test') eb['custom_key'] = "not so default" self.assertEqual(eb['custom_key'], 'not so default') # test if extra toolchain options are being passed self.assertEqual(eb.toolchain.options['static'], True) # test extra mandatory parameters extra_vars.update({'mandatory_key': ['default', 'another mandatory key', easyconfig.MANDATORY]}) self.assertErrorRegex(EasyBuildError, r"mandatory parameters not provided", EasyConfig, self.eb_file, extra_options=extra_vars) self.contents += '\nmandatory_key = "value"' self.prep() eb = EasyConfig(self.eb_file, extra_options=extra_vars) self.assertEqual(eb['mandatory_key'], 'value')
def test_parse_yeb(self): """Test parsing of .yeb easyconfigs.""" if 'yaml' not in sys.modules: print "Skipping test_parse_yeb (no PyYAML available)" return build_options = { 'check_osdeps': False, 'external_modules_metadata': {}, 'valid_module_classes': module_classes(), } init_config(build_options=build_options) easybuild.tools.build_log.EXPERIMENTAL = True testdir = os.path.dirname(os.path.abspath(__file__)) test_easyconfigs = os.path.join(testdir, 'easyconfigs') test_yeb_easyconfigs = os.path.join(testdir, 'easyconfigs', 'yeb') # test parsing test_files = [ 'bzip2-1.0.6-GCC-4.9.2', 'gzip-1.6-GCC-4.9.2', 'goolf-1.4.10', 'ictce-4.1.13', 'SQLite-3.8.10.2-goolf-1.4.10', 'Python-2.7.10-ictce-4.1.13', 'CrayCCE-5.1.29', 'toy-0.0', ] for filename in test_files: ec_yeb = EasyConfig( os.path.join(test_yeb_easyconfigs, '%s.yeb' % filename)) # compare with parsed result of .eb easyconfig ec_file = glob.glob( os.path.join(test_easyconfigs, 'test_ecs', '*', '*', '%s.eb' % filename))[0] ec_eb = EasyConfig(ec_file) no_match = False for key in sorted(ec_yeb.asdict()): eb_val = ec_eb[key] yeb_val = ec_yeb[key] if key == 'description': # multi-line string is always terminated with '\n' in YAML, so strip it off yeb_val = yeb_val.strip() self.assertEqual(yeb_val, eb_val)
def test_package(self): """Test package function.""" init_config(build_options={'silent': True}) test_easyconfigs = os.path.join( os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') ec = EasyConfig(os.path.join(test_easyconfigs, 'toy-0.0-gompi-1.3.12-test.eb'), validate=False) mock_fpm(self.test_prefix) # import needs to be done here, since test easyblocks are only included later from easybuild.easyblocks.toy import EB_toy easyblock = EB_toy(ec) # build & install first easyblock.run_all_steps(False) # package using default packaging configuration (FPM to build RPM packages) pkgdir = package(easyblock) pkgfile = os.path.join( pkgdir, 'toy-0.0-gompi-1.3.12-test-eb-%s.1.rpm' % EASYBUILD_VERSION) self.assertTrue(os.path.isfile(pkgfile), "Found %s" % pkgfile) pkgtxt = read_file(pkgfile) pkgtxt_regex = re.compile("Contents of installdir %s" % easyblock.installdir) self.assertTrue( pkgtxt_regex.search(pkgtxt), "Pattern '%s' found in: %s" % (pkgtxt_regex.pattern, pkgtxt))
def test_mandatory(self): """ make sure all checking of mandatory variables works """ self.contents = '\n'.join([ 'name = "pi"', 'version = "3.14"', ]) self.prep() self.assertErrorRegex(EasyBuildError, "mandatory variables .* not provided", EasyConfig, self.eb_file) self.contents += '\n' + '\n'.join([ 'homepage = "http://google.com"', 'description = "test easyconfig"', 'toolchain = {"name": "dummy", "version": "dummy"}', ]) self.prep() eb = EasyConfig(self.eb_file, valid_stops=self.all_stops) self.assertEqual(eb['name'], "pi") self.assertEqual(eb['version'], "3.14") self.assertEqual(eb['homepage'], "http://google.com") self.assertEqual(eb['toolchain'], { "name": "dummy", "version": "dummy" }) self.assertEqual(eb['description'], "test easyconfig")
def test_exts_list(self): """Test handling of list of extensions.""" os.environ['EASYBUILD_SOURCEPATH'] = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') init_config() self.contents = '\n'.join([ 'name = "pi"', 'version = "3.14"', 'homepage = "http://example.com"', 'description = "test easyconfig"', 'toolchain = {"name": "dummy", "version": "dummy"}', 'exts_list = [', ' ("ext1", "ext_ver1", {', ' "source_tmpl": "gzip-1.4.eb",', # dummy source template to avoid downloading fail ' "source_urls": ["http://example.com/"]', ' }),', ' ("ext2", "ext_ver2", {', ' "source_tmpl": "gzip-1.4.eb",', # dummy source template to avoid downloading fail ' "source_urls": [("http://example.com", "suffix")],' ' "patches": ["toy-0.0.eb"],', # dummy patch to avoid downloading fail ' "checksums": [', ' "504c7036558938f997c1c269a01d7458",', # checksum for source (gzip-1.4.eb) ' "ddd5161154f5db67701525123129ff09",', # checksum for patch (toy-0.0.eb) ' ],', ' }),', ']', ]) self.prep() ec = EasyConfig(self.eb_file) eb = EasyBlock(ec) exts_sources = eb.fetch_extension_sources()
def runTest(self): """ test easyconfig templating """ eb = EasyConfig(self.eb_file, validate=False, valid_stops=self.all_stops) eb.validate() eb.generate_template_values() self.assertEqual(eb['description'], "test easyconfig PI") const_dict = dict([(x[0], x[1]) for x in easyconfig.templates.TEMPLATE_CONSTANTS]) self.assertEqual(eb['sources'][0], const_dict['SOURCE_TAR_GZ'] % self.inp) self.assertEqual(eb['sources'][1][0], const_dict['SOURCELOWER_TAR_GZ'] % self.inp) self.assertEqual(eb['sources'][1][1], 'tar xfvz %s') self.assertEqual(eb['source_urls'][0], const_dict['GOOGLECODE_SOURCE'] % self.inp) # test the escaping insanity here (ie all the crap we allow in easyconfigs) eb['description'] = "test easyconfig % %% %s% %%% %(name)s %%(name)s %%%(name)s %%%%(name)s" self.assertEqual(eb['description'], "test easyconfig % %% %s% %%% PI %(name)s %PI %%(name)s")
def test_toolchain(self): """Test whether toolchain is initialized correctly.""" ec_file = find_full_path( os.path.join('test', 'framework', 'easyconfigs', 'gzip-1.4.eb')) ec = EasyConfig(ec_file, validate=False) tc = ec.toolchain self.assertTrue('debug' in tc.options)
def test_skip_extensions_step(self): """Test the skip_extensions_step""" self.contents = '\n'.join([ 'easyblock = "ConfigureMake"', 'name = "pi"', 'version = "3.14"', 'homepage = "http://example.com"', 'description = "test easyconfig"', 'toolchain = {"name": "dummy", "version": "dummy"}', 'exts_list = ["ext1", "ext2"]', 'exts_filter = ("if [ %(ext_name)s == \'ext2\' ]; then exit 0; else exit 1; fi", "")', 'exts_defaultclass = "DummyExtension"', ]) # check if skip skips correct extensions self.writeEC() eb = EasyBlock(EasyConfig(self.eb_file)) eb.builddir = config.build_path() eb.installdir = config.install_path() eb.skip = True eb.extensions_step(fetch=True) # 'ext1' should be in eb.exts self.assertTrue('ext1' in [y for x in eb.exts for y in x.values()]) # 'ext2' should not self.assertFalse('ext2' in [y for x in eb.exts for y in x.values()]) # cleanup eb.close_log() os.remove(eb.logfile)
def test_make_module_req(self): """Testcase for make_module_req""" self.contents = '\n'.join([ 'name = "pi"', 'version = "3.14"', 'homepage = "http://example.com"', 'description = "test easyconfig"', 'toolchain = {"name":"dummy", "version": "dummy"}', ]) self.writeEC() eb = EasyBlock(EasyConfig(self.eb_file)) eb.installdir = config.install_path() # create fake directories and files that should be guessed os.makedirs(eb.installdir) open(os.path.join(eb.installdir, 'foo.jar'), 'w').write('foo.jar') open(os.path.join(eb.installdir, 'bla.jar'), 'w').write('bla.jar') os.mkdir(os.path.join(eb.installdir, 'bin')) os.mkdir(os.path.join(eb.installdir, 'share')) os.mkdir(os.path.join(eb.installdir, 'share', 'man')) # this is not a path that should be picked up os.mkdir(os.path.join(eb.installdir, 'CPATH')) guess = eb.make_module_req() self.assertTrue(re.search("^prepend-path\s+CLASSPATH\s+\$root/bla.jar$", guess, re.M)) self.assertTrue(re.search("^prepend-path\s+CLASSPATH\s+\$root/foo.jar$", guess, re.M)) self.assertTrue(re.search("^prepend-path\s+MANPATH\s+\$root/share/man$", guess, re.M)) self.assertTrue(re.search("^prepend-path\s+PATH\s+\$root/bin$", guess, re.M)) self.assertFalse(re.search("^prepend-path\s+CPATH\s+.*$", guess, re.M)) # cleanup eb.close_log() os.remove(eb.logfile)
def process_easyconfig(path, onlyBlocks=None, regtest_online=False, validate=True): """ Process easyconfig, returning some information for each block """ blocks = retrieve_blocks_in_spec(path, onlyBlocks) easyconfigs = [] for spec in blocks: # process for dependencies and real installversionname # - use mod? __init__ and importCfg are ignored. log.debug("Processing easyconfig %s" % spec) # create easyconfig try: all_stops = [x[0] for x in EasyBlock.get_steps()] ec = EasyConfig(spec, validate=validate, valid_module_classes=module_classes(), valid_stops=all_stops) except EasyBuildError, err: msg = "Failed to process easyconfig %s:\n%s" % (spec, err.msg) log.exception(msg) name = ec['name'] # this app will appear as following module in the list easyconfig = { 'spec': spec, 'module': (ec.name, ec.get_installversion()), 'dependencies': [] } if len(blocks) > 1: easyconfig['originalSpec'] = path for d in ec.dependencies(): dep = (d['name'], d['tc']) log.debug("Adding dependency %s for app %s." % (dep, name)) easyconfig['dependencies'].append(dep) if ec.toolchain.name != 'dummy': dep = (ec.toolchain.name, ec.toolchain.version) log.debug("Adding toolchain %s as dependency for app %s." % (dep, name)) easyconfig['dependencies'].append(dep) del ec # this is used by the parallel builder easyconfig['unresolvedDependencies'] = copy.copy(easyconfig['dependencies']) easyconfigs.append(easyconfig)
def test_hierarchical_mns(self): """Test hierarchical module naming scheme.""" ecs_dir = os.path.join(os.path.dirname(__file__), 'easyconfigs') all_stops = [x[0] for x in EasyBlock.get_steps()] build_options = { 'check_osdeps': False, 'robot_path': [ecs_dir], 'valid_stops': all_stops, 'validate': False, } os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'HierarchicalMNS' init_config(build_options=build_options) ec = EasyConfig(os.path.join(ecs_dir, 'GCC-4.7.2.eb')) self.assertEqual(ActiveMNS().det_full_module_name(ec), 'Core/GCC/4.7.2') self.assertEqual(ActiveMNS().det_short_module_name(ec), 'GCC/4.7.2') self.assertEqual(ActiveMNS().det_module_subdir(ec), 'Core') self.assertEqual(ActiveMNS().det_modpath_extensions(ec), ['Compiler/GCC/4.7.2']) self.assertEqual(ActiveMNS().det_init_modulepaths(ec), ['Core']) ec = EasyConfig(os.path.join(ecs_dir, 'OpenMPI-1.6.4-GCC-4.7.2.eb')) self.assertEqual(ActiveMNS().det_full_module_name(ec), 'Compiler/GCC/4.7.2/OpenMPI/1.6.4') self.assertEqual(ActiveMNS().det_short_module_name(ec), 'OpenMPI/1.6.4') self.assertEqual(ActiveMNS().det_module_subdir(ec), 'Compiler/GCC/4.7.2') self.assertEqual(ActiveMNS().det_modpath_extensions(ec), ['MPI/GCC/4.7.2/OpenMPI/1.6.4']) self.assertEqual(ActiveMNS().det_init_modulepaths(ec), ['Core']) ec = EasyConfig(os.path.join(ecs_dir, 'gzip-1.5-goolf-1.4.10.eb')) self.assertEqual(ActiveMNS().det_full_module_name(ec), 'MPI/GCC/4.7.2/OpenMPI/1.6.4/gzip/1.5') self.assertEqual(ActiveMNS().det_short_module_name(ec), 'gzip/1.5') self.assertEqual(ActiveMNS().det_module_subdir(ec), 'MPI/GCC/4.7.2/OpenMPI/1.6.4') self.assertEqual(ActiveMNS().det_modpath_extensions(ec), []) self.assertEqual(ActiveMNS().det_init_modulepaths(ec), ['Core']) os.environ[ 'EASYBUILD_MODULE_NAMING_SCHEME'] = self.orig_module_naming_scheme init_config(build_options=build_options)
def test_parse_yeb(self): """Test parsing of .yeb easyconfigs.""" if 'yaml' not in sys.modules: print "Skipping test_parse_yeb (no PyYAML available)" return build_options = { 'check_osdeps': False, 'external_modules_metadata': {}, 'valid_module_classes': module_classes(), } init_config(build_options=build_options) easybuild.tools.build_log.EXPERIMENTAL = True testdir = os.path.dirname(os.path.abspath(__file__)) test_easyconfigs = os.path.join(testdir, 'easyconfigs') test_yeb_easyconfigs = os.path.join(testdir, 'easyconfigs', 'yeb') # test parsing test_files = [ 'bzip2-1.0.6-GCC-4.9.2', 'gzip-1.6-GCC-4.9.2', 'goolf-1.4.10', 'ictce-4.1.13', 'SQLite-3.8.10.2-goolf-1.4.10', 'Python-2.7.10-ictce-4.1.13', 'CrayCCE-5.1.29', 'toy-0.0', ] for filename in test_files: ec_yeb = EasyConfig(os.path.join(test_yeb_easyconfigs, '%s.yeb' % filename)) # compare with parsed result of .eb easyconfig ec_file = glob.glob(os.path.join(test_easyconfigs, 'test_ecs', '*', '*', '%s.eb' % filename))[0] ec_eb = EasyConfig(ec_file) no_match = False for key in sorted(ec_yeb.asdict()): eb_val = ec_eb[key] yeb_val = ec_yeb[key] if key == 'description': # multi-line string is always terminated with '\n' in YAML, so strip it off yeb_val = yeb_val.strip() self.assertEqual(yeb_val, eb_val)
def test_make_module_step(self): """Test the make_module_step""" name = "pi" version = "3.14" deps = [('GCC', '4.6.4')] hiddendeps = [('toy', '0.0-deps')] alldeps = deps + hiddendeps # hidden deps must be included in list of deps modextravars = {'PI': '3.1415', 'FOO': 'bar'} modextrapaths = {'PATH': 'pibin', 'CPATH': 'pi/include'} self.contents = '\n'.join([ 'easyblock = "ConfigureMake"', 'name = "%s"' % name, 'version = "%s"' % version, 'homepage = "http://example.com"', 'description = "test easyconfig"', "toolchain = {'name': 'dummy', 'version': 'dummy'}", "dependencies = %s" % str(alldeps), "hiddendependencies = %s" % str(hiddendeps), "builddependencies = [('OpenMPI', '1.6.4-GCC-4.6.4')]", "modextravars = %s" % str(modextravars), "modextrapaths = %s" % str(modextrapaths), ]) test_dir = os.path.dirname(os.path.abspath(__file__)) os.environ['MODULEPATH'] = os.path.join(test_dir, 'modules') # test if module is generated correctly self.writeEC() ec = EasyConfig(self.eb_file) eb = EasyBlock(ec) eb.installdir = os.path.join(config.install_path(), 'pi', '3.14') eb.check_readiness_step() modpath = os.path.join(eb.make_module_step(), name, version) self.assertTrue(os.path.exists(modpath), "%s exists" % modpath) # verify contents of module f = open(modpath, 'r') txt = f.read() f.close() self.assertTrue(re.search("^#%Module", txt.split('\n')[0])) self.assertTrue(re.search("^conflict\s+%s$" % name, txt, re.M)) self.assertTrue(re.search("^set\s+root\s+%s$" % eb.installdir, txt, re.M)) self.assertTrue(re.search('^setenv\s+EBROOT%s\s+".root"\s*$' % name.upper(), txt, re.M)) self.assertTrue(re.search('^setenv\s+EBVERSION%s\s+"%s"$' % (name.upper(), version), txt, re.M)) for (key, val) in modextravars.items(): regex = re.compile('^setenv\s+%s\s+"%s"$' % (key, val), re.M) self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt)) for (key, val) in modextrapaths.items(): regex = re.compile('^prepend-path\s+%s\s+\$root/%s$' % (key, val), re.M) self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt)) for (name, ver) in deps: regex = re.compile('^\s*module load %s\s*$' % os.path.join(name, ver), re.M) self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt)) for (name, ver) in hiddendeps: regex = re.compile('^\s*module load %s/.%s\s*$' % (name, ver), re.M) self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt))
def test_ec(ecfile, short_modname, mod_subdir, modpath_exts, user_modpath_exts, init_modpaths): """Test whether active module naming scheme returns expected values.""" ec = EasyConfig(glob.glob(os.path.join(ecs_dir, '*','*', ecfile))[0]) self.assertEqual(ActiveMNS().det_full_module_name(ec), os.path.join(mod_subdir, short_modname)) self.assertEqual(ActiveMNS().det_short_module_name(ec), short_modname) self.assertEqual(ActiveMNS().det_module_subdir(ec), mod_subdir) self.assertEqual(ActiveMNS().det_modpath_extensions(ec), modpath_exts) self.assertEqual(ActiveMNS().det_user_modpath_extensions(ec), user_modpath_exts) self.assertEqual(ActiveMNS().det_init_modulepaths(ec), init_modpaths)
def template_module_only_test(self, easyblock, name='foo', version='1.3.2', extra_txt=''): """Test whether all easyblocks are compatible with --module-only.""" class_regex = re.compile("^class (.*)\(.*", re.M) self.log.debug("easyblock: %s" % easyblock) # read easyblock Python module f = open(easyblock, "r") txt = f.read() f.close() # obtain easyblock class name using regex res = class_regex.search(txt) if res: ebname = res.group(1) self.log.debug("Found class name for easyblock %s: %s" % (easyblock, ebname)) # figure out list of mandatory variables, and define with dummy values as necessary app_class = get_easyblock_class(ebname) # extend easyconfig to make sure mandatory custom easyconfig paramters are defined extra_options = app_class.extra_options() for (key, val) in extra_options.items(): if val[2] == MANDATORY: extra_txt += '%s = "foo"\n' % key # write easyconfig file self.writeEC(ebname, name=name, version=version, extratxt=extra_txt) # initialize easyblock # if this doesn't fail, the test succeeds app = app_class(EasyConfig(self.eb_file)) # run all steps, most should be skipped orig_workdir = os.getcwd() try: app.run_all_steps(run_test_cases=False) finally: os.chdir(orig_workdir) modfile = os.path.join(TMPDIR, 'modules', 'all', 'foo', '1.3.2') luamodfile = '%s.lua' % modfile self.assertTrue( os.path.exists(modfile) or os.path.exists(luamodfile), "Module file %s or %s was generated" % (modfile, luamodfile)) # cleanup app.close_log() os.remove(app.logfile) else: self.assertTrue(False, "Class found in easyblock %s" % easyblock)
def template_init_test(self, easyblock): """Test whether all easyconfigs can be initialized.""" def check_extra_options_format(extra_options): """Make sure extra_options value is of correct format.""" # EasyBuild v1.x self.assertTrue(isinstance(extra_options, list)) for extra_option in extra_options: self.assertTrue(isinstance(extra_option, tuple)) self.assertEqual(len(extra_option), 2) self.assertTrue(isinstance(extra_option[0], basestring)) self.assertTrue(isinstance(extra_option[1], list)) self.assertEqual(len(extra_option[1]), 3) # EasyBuild v2.0 (breaks backward compatibility compared to v1.x) #self.assertTrue(isinstance(extra_options, dict)) #for key in extra_options: # self.assertTrue(isinstance(extra_options[key], list)) # self.assertTrue(len(extra_options[key]), 3) class_regex = re.compile("^class (.*)\(.*", re.M) self.log.debug("easyblock: %s" % easyblock) # obtain easyblock class name using regex f = open(easyblock, "r") txt = f.read() f.close() res = class_regex.search(txt) if res: ebname = res.group(1) self.log.debug("Found class name for easyblock %s: %s" % (easyblock, ebname)) # figure out list of mandatory variables, and define with dummy values as necessary app_class = get_easyblock_class(ebname) extra_options = app_class.extra_options() check_extra_options_format(extra_options) # extend easyconfig to make sure mandatory custom easyconfig paramters are defined extra_txt = '' for (key, val) in extra_options: if val[2] == MANDATORY: extra_txt += '%s = "foo"\n' % key # write easyconfig file self.writeEC(ebname, extra_txt) # initialize easyblock # if this doesn't fail, the test succeeds app = app_class(EasyConfig(self.eb_file)) # cleanup app.close_log() os.remove(app.logfile) else: self.assertTrue(False, "Class found in easyblock %s" % easyblock)
def test_external_module_toolchain(self): """Test specifying external (build) dependencies.""" ecpath = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'yeb', 'CrayCCE-5.1.29.yeb') metadata = { 'name': ['foo', 'bar'], 'version': ['1.2.3', '3.2.1'], 'prefix': '/foo/bar', } build_options = { 'external_modules_metadata': {'fftw/3.3.4.0': metadata}, 'valid_module_classes': module_classes(), } init_config(build_options=build_options) easybuild.tools.build_log.EXPERIMENTAL = True ec = EasyConfig(ecpath) self.assertEqual(ec.dependencies()[1]['full_mod_name'], 'fftw/3.3.4.0') self.assertEqual(ec.dependencies()[1]['external_module_metadata'], metadata)
def ec_filename_for(path): """ Return a suiting file name for the easyconfig file at <path>, as determined by its contents. """ ec = EasyConfig(path, validate=False) fn = "%s-%s.eb" % (ec['name'], det_full_ec_version(ec)) return fn
def test_exclude_path_to_top_of_module_tree(self): """ Make sure that modules under the HierarchicalMNS are correct, w.r.t. not including any load statements for modules that build up the path to the top of the module tree. """ self.orig_module_naming_scheme = config.get_module_naming_scheme() test_ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') all_stops = [x[0] for x in EasyBlock.get_steps()] build_options = { 'check_osdeps': False, 'robot_path': [test_ecs_path], 'valid_stops': all_stops, 'validate': False, } os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'HierarchicalMNS' init_config(build_options=build_options) self.setup_hierarchical_modules() modfile_prefix = os.path.join(self.test_installpath, 'modules', 'all') mkdir(os.path.join(modfile_prefix, 'Compiler', 'GCC', '4.8.3'), parents=True) mkdir(os.path.join(modfile_prefix, 'MPI', 'intel', '2013.5.192-GCC-4.8.3', 'impi', '4.1.3.049'), parents=True) impi_modfile_path = os.path.join('Compiler', 'intel', '2013.5.192-GCC-4.8.3', 'impi', '4.1.3.049') imkl_modfile_path = os.path.join('MPI', 'intel', '2013.5.192-GCC-4.8.3', 'impi', '4.1.3.049', 'imkl', '11.1.2.144') if get_module_syntax() == 'Lua': impi_modfile_path += '.lua' imkl_modfile_path += '.lua' # example: for imkl on top of iimpi toolchain with HierarchicalMNS, no module load statements should be included # not for the toolchain or any of the toolchain components, # since both icc/ifort and impi form the path to the top of the module tree tests = [ ('impi-4.1.3.049-iccifort-2013.5.192-GCC-4.8.3.eb', impi_modfile_path, ['icc', 'ifort', 'iccifort']), ('imkl-11.1.2.144-iimpi-5.5.3-GCC-4.8.3.eb', imkl_modfile_path, ['icc', 'ifort', 'impi', 'iccifort', 'iimpi']), ] for ec_file, modfile_path, excluded_deps in tests: ec = EasyConfig(os.path.join(test_ecs_path, ec_file)) eb = EasyBlock(ec) eb.toolchain.prepare() modpath = eb.make_module_step() modfile_path = os.path.join(modpath, modfile_path) modtxt = read_file(modfile_path) for dep in excluded_deps: tup = (dep, modfile_path, modtxt) failmsg = "No 'module load' statement found for '%s' not found in module %s: %s" % tup if get_module_syntax() == 'Tcl': self.assertFalse(re.search('module load %s' % dep, modtxt), failmsg) elif get_module_syntax() == 'Lua': self.assertFalse(re.search('load("%s")' % dep, modtxt), failmsg) else: self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = self.orig_module_naming_scheme init_config(build_options=build_options)
def test_dependency(self): """ test all possible ways of specifying dependencies """ self.contents = '\n'.join([ 'name = "pi"', 'version = "3.14"', 'homepage = "http://google.com"', 'description = "test easyconfig"', 'toolchain = {"name":"GCC", "version": "4.6.3"}', 'dependencies = [("first", "1.1"), {"name": "second", "version": "2.2"}]', 'builddependencies = [("first", "1.1"), {"name": "second", "version": "2.2"}]', ]) self.prep() eb = EasyConfig(self.eb_file, valid_stops=self.all_stops) # should include builddependencies self.assertEqual(len(eb.dependencies()), 4) self.assertEqual(len(eb.builddependencies()), 2) first = eb.dependencies()[0] second = eb.dependencies()[1] self.assertEqual(first['name'], "first") self.assertEqual(second['name'], "second") self.assertEqual(first['version'], "1.1") self.assertEqual(second['version'], "2.2") self.assertEqual(first['tc'], '1.1-GCC-4.6.3') self.assertEqual(second['tc'], '2.2-GCC-4.6.3') # same tests for builddependencies first = eb.builddependencies()[0] second = eb.builddependencies()[1] self.assertEqual(first['name'], "first") self.assertEqual(second['name'], "second") self.assertEqual(first['version'], "1.1") self.assertEqual(second['version'], "2.2") self.assertEqual(first['tc'], '1.1-GCC-4.6.3') self.assertEqual(second['tc'], '2.2-GCC-4.6.3') eb['dependencies'] = ["wrong type"] self.assertErrorRegex(EasyBuildError, "wrong type from unsupported type", eb.dependencies) eb['dependencies'] = [()] self.assertErrorRegex(EasyBuildError, "without name", eb.dependencies) eb['dependencies'] = [{'name': "test"}] self.assertErrorRegex(EasyBuildError, "without version", eb.dependencies)
def test_toolchain_inspection(self): """Test whether available toolchain inspection functionality is working.""" build_options = { 'robot_path': os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs'), 'valid_module_classes': module_classes(), } init_config(build_options=build_options) ec = EasyConfig(os.path.join(os.path.dirname(__file__), 'easyconfigs', 'gzip-1.5-goolf-1.4.10.eb')) self.assertEqual(['/'.join([x['name'], x['version']]) for x in det_toolchain_compilers(ec)], ['GCC/4.7.2']) self.assertEqual(det_toolchain_mpi(ec)['name'], 'OpenMPI') ec = EasyConfig(os.path.join(os.path.dirname(__file__), 'easyconfigs', 'hwloc-1.6.2-GCC-4.6.4.eb')) tc_comps = det_toolchain_compilers(ec) self.assertEqual(['/'.join([x['name'], x['version']]) for x in tc_comps], ['GCC/4.6.4']) self.assertEqual(det_toolchain_mpi(ec), None) ec = EasyConfig(os.path.join(os.path.dirname(__file__), 'easyconfigs', 'toy-0.0.eb')) self.assertEqual(det_toolchain_compilers(ec), None) self.assertEqual(det_toolchain_mpi(ec), None)
def test_make_module_extra(self): """Test for make_module_extra.""" self.contents = '\n'.join([ 'easyblock = "ConfigureMake"', 'name = "pi"', 'version = "3.14"', 'homepage = "http://example.com"', 'description = "test easyconfig"', "toolchain = {'name': 'gompi', 'version': '1.1.0-no-OFED'}", 'dependencies = [', " ('FFTW', '3.3.1'),", " ('LAPACK', '3.4.0'),", ']', ]) self.writeEC() eb = EasyBlock(EasyConfig(self.eb_file)) eb.installdir = os.path.join(config.install_path(), 'pi', '3.14') if get_module_syntax() == 'Tcl': expected_default = re.compile(r'\n'.join([ r'setenv\s+EBROOTPI\s+\"\$root"', r'setenv\s+EBVERSIONPI\s+"3.14"', r'setenv\s+EBDEVELPI\s+"\$root/easybuild/pi-3.14-gompi-1.1.0-no-OFED-easybuild-devel"', ])) expected_alt = re.compile(r'\n'.join([ r'setenv\s+EBROOTPI\s+"/opt/software/tau/6.28"', r'setenv\s+EBVERSIONPI\s+"6.28"', r'setenv\s+EBDEVELPI\s+"\$root/easybuild/pi-3.14-gompi-1.1.0-no-OFED-easybuild-devel"', ])) elif get_module_syntax() == 'Lua': expected_default = re.compile(r'\n'.join([ r'setenv\("EBROOTPI", root\)', r'setenv\("EBVERSIONPI", "3.14"\)', r'setenv\("EBDEVELPI", pathJoin\(root, "easybuild/pi-3.14-gompi-1.1.0-no-OFED-easybuild-devel"\)\)', ])) expected_alt = re.compile(r'\n'.join([ r'setenv\("EBROOTPI", "/opt/software/tau/6.28"\)', r'setenv\("EBVERSIONPI", "6.28"\)', r'setenv\("EBDEVELPI", pathJoin\(root, "easybuild/pi-3.14-gompi-1.1.0-no-OFED-easybuild-devel"\)\)', ])) else: self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) defaulttxt = eb.make_module_extra().strip() self.assertTrue( expected_default.match(defaulttxt), "Pattern %s found in %s" % (expected_default.pattern, defaulttxt)) alttxt = eb.make_module_extra(altroot='/opt/software/tau/6.28', altversion='6.28').strip() self.assertTrue( expected_alt.match(alttxt), "Pattern %s found in %s" % (expected_alt.pattern, alttxt))
def test_active_pns(self): """Test use of ActivePNS.""" test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs') ec = EasyConfig(os.path.join(test_easyconfigs, 'OpenMPI-1.6.4-GCC-4.6.4.eb'), validate=False) pns = ActivePNS() # default: EasyBuild package naming scheme, pkg release 1 self.assertEqual(pns.name(ec), 'OpenMPI-1.6.4-GCC-4.6.4') self.assertEqual(pns.version(ec), 'eb-%s' % EASYBUILD_VERSION) self.assertEqual(pns.release(ec), '1')
def test_package(self): """Test package function.""" init_config(build_options={'silent': True}) topdir = os.path.dirname(os.path.abspath(__file__)) test_easyconfigs = os.path.join(topdir, 'easyconfigs', 'test_ecs') ec = EasyConfig(os.path.join(test_easyconfigs, 't', 'toy', 'toy-0.0-gompi-1.3.12-test.eb'), validate=False) mock_fpm(self.test_prefix) # import needs to be done here, since test easyblocks are only included later from easybuild.easyblocks.toy import EB_toy easyblock = EB_toy(ec) # build & install first easyblock.run_all_steps(False) # write a dummy log and report file to make sure they don't get packaged logfile = os.path.join(easyblock.installdir, log_path(), "logfile.log") write_file(logfile, "I'm a logfile") reportfile = os.path.join(easyblock.installdir, log_path(), "report.md") write_file(reportfile, "I'm a reportfile") # package using default packaging configuration (FPM to build RPM packages) pkgdir = package(easyblock) pkgfile = os.path.join( pkgdir, 'toy-0.0-gompi-1.3.12-test-eb-%s.1.rpm' % EASYBUILD_VERSION) self.assertTrue(os.path.isfile(pkgfile), "Found %s" % pkgfile) pkgtxt = read_file(pkgfile) pkgtxt_regex = re.compile("STARTCONTENTS of installdir %s" % easyblock.installdir) self.assertTrue( pkgtxt_regex.search(pkgtxt), "Pattern '%s' found in: %s" % (pkgtxt_regex.pattern, pkgtxt)) no_logfiles_regex = re.compile( r'STARTCONTENTS.*\.(log|md)$.*ENDCONTENTS', re.DOTALL | re.MULTILINE) self.assertFalse( no_logfiles_regex.search(pkgtxt), "Pattern not '%s' found in: %s" % (no_logfiles_regex.pattern, pkgtxt)) if DEBUG: print "The FPM script debug output" print read_file(os.path.join(self.test_prefix, DEBUG_FPM_FILE)) print "The Package File" print read_file(pkgfile)
def runTest(self): """ test other validations beside mandatory variables """ eb = EasyConfig(self.eb_file, validate=False, valid_stops=self.all_stops) self.assertErrorRegex(EasyBuildError, r"\w* provided '\w*' is not valid", eb.validate) eb['stop'] = 'patch' # this should now not crash eb.validate() eb['osdependencies'] = ['non-existent-dep'] self.assertErrorRegex(EasyBuildError, "OS dependencies were not found", eb.validate) # dummy toolchain, installversion == version self.assertEqual(eb.get_installversion(), "3.14") os.chmod(self.eb_file, 0000) self.assertErrorRegex(EasyBuildError, "Unexpected IOError", EasyConfig, self.eb_file) os.chmod(self.eb_file, 0755) self.contents += "\nsyntax_error'" self.setUp() self.assertErrorRegex(EasyBuildError, "SyntaxError", EasyConfig, self.eb_file)
def test_dependency(self): """ test all possible ways of specifying dependencies """ self.contents = '\n'.join([ 'easyblock = "ConfigureMake"', 'name = "pi"', 'version = "3.14"', 'homepage = "http://example.com"', 'description = "test easyconfig"', 'toolchain = {"name":"GCC", "version": "4.6.3"}', 'dependencies = [("first", "1.1"), {"name": "second", "version": "2.2"}]', 'builddependencies = [("first", "1.1"), {"name": "second", "version": "2.2"}]', ]) self.prep() eb = EasyConfig(self.eb_file) # should include builddependencies self.assertEqual(len(eb.dependencies()), 4) self.assertEqual(len(eb.builddependencies()), 2) first = eb.dependencies()[0] second = eb.dependencies()[1] self.assertEqual(first['name'], "first") self.assertEqual(second['name'], "second") self.assertEqual(first['version'], "1.1") self.assertEqual(second['version'], "2.2") self.assertEqual(det_full_ec_version(first), '1.1-GCC-4.6.3') self.assertEqual(det_full_ec_version(second), '2.2-GCC-4.6.3') # same tests for builddependencies first = eb.builddependencies()[0] second = eb.builddependencies()[1] self.assertEqual(first['name'], "first") self.assertEqual(second['name'], "second") self.assertEqual(first['version'], "1.1") self.assertEqual(second['version'], "2.2") self.assertEqual(det_full_ec_version(first), '1.1-GCC-4.6.3') self.assertEqual(det_full_ec_version(second), '2.2-GCC-4.6.3') self.assertErrorRegex(EasyBuildError, "Dependency foo of unsupported type", eb._parse_dependency, "foo") self.assertErrorRegex(EasyBuildError, "without name", eb._parse_dependency, ()) self.assertErrorRegex(EasyBuildError, "without version", eb._parse_dependency, {'name': 'test'}) err_msg = "Incorrect external dependency specification" self.assertErrorRegex(EasyBuildError, err_msg, eb._parse_dependency, (EXTERNAL_MODULE_MARKER,)) self.assertErrorRegex(EasyBuildError, err_msg, eb._parse_dependency, ('foo', '1.2.3', EXTERNAL_MODULE_MARKER))
def runTest(self): ver = "1.2.3" verpref = "myprefix" versuff = "mysuffix" tcname = "mytc" tcver = "4.1.2" extra_patches = ['t5.patch', 't6.patch'] homepage = "http://www.justatest.com" tweaks = { 'version': ver, 'versionprefix': verpref, 'versionsuffix': versuff, 'toolchain_version': tcver, 'patches': extra_patches } tweak(self.eb_file, self.tweaked_fn, tweaks) eb = EasyConfig(self.tweaked_fn, valid_stops=self.all_stops) self.assertEqual(eb['version'], ver) self.assertEqual(eb['versionprefix'], verpref) self.assertEqual(eb['versionsuffix'], versuff) self.assertEqual(eb['toolchain']['version'], tcver) self.assertEqual(eb['patches'], extra_patches + self.patches) eb = EasyConfig(self.eb_file, valid_stops=self.all_stops) # eb['toolchain']['version'] = tcver does not work as expected with templating enabled eb.enable_templating = False eb['version'] = ver eb['toolchain']['version'] = tcver eb.enable_templating = True eb.dump(self.eb_file) tweaks = { 'toolchain_name': tcname, 'patches': extra_patches[0:1], 'homepage': homepage, 'foo': "bar" } tweak(self.eb_file, self.tweaked_fn, tweaks) eb = EasyConfig(self.tweaked_fn, valid_stops=self.all_stops) self.assertEqual(eb['toolchain']['name'], tcname) self.assertEqual(eb['toolchain']['version'], tcver) self.assertEqual(eb['patches'], extra_patches[0:1] + self.patches) self.assertEqual(eb['version'], ver) self.assertEqual(eb['homepage'], homepage)
def test_external_dependencies(self): """Test specifying external (build) dependencies.""" ectxt = read_file(os.path.join(os.path.dirname(__file__), 'easyconfigs', 'toy-0.0-deps.eb')) toy_ec = os.path.join(self.test_prefix, 'toy-0.0-external-deps.eb') # just specify some of the test modules we ship, doesn't matter where they come from ectxt += "\ndependencies += [('foobar/1.2.3', EXTERNAL_MODULE)]" ectxt += "\nbuilddependencies = [('somebuilddep/0.1', EXTERNAL_MODULE)]" write_file(toy_ec, ectxt) build_options = { 'valid_module_classes': module_classes(), 'external_modules_metadata': ConfigObj(), } init_config(build_options=build_options) ec = EasyConfig(toy_ec) builddeps = ec.builddependencies() self.assertEqual(len(builddeps), 1) self.assertEqual(builddeps[0]['short_mod_name'], 'somebuilddep/0.1') self.assertEqual(builddeps[0]['full_mod_name'], 'somebuilddep/0.1') self.assertEqual(builddeps[0]['external_module'], True) deps = ec.dependencies() self.assertEqual(len(deps), 4) correct_deps = ['ictce/4.1.13', 'GCC/4.7.2', 'foobar/1.2.3', 'somebuilddep/0.1'] self.assertEqual([d['short_mod_name'] for d in deps], correct_deps) self.assertEqual([d['full_mod_name'] for d in deps], correct_deps) self.assertEqual([d['external_module'] for d in deps], [False, True, True, True]) metadata = os.path.join(self.test_prefix, 'external_modules_metadata.cfg') metadatatxt = '\n'.join(['[foobar/1.2.3]', 'name = foo,bar', 'version = 1.2.3,3.2.1', 'prefix = /foo/bar']) write_file(metadata, metadatatxt) cfg = init_config(args=['--external-modules-metadata=%s' % metadata]) build_options = { 'external_modules_metadata': cfg.external_modules_metadata, 'valid_module_classes': module_classes(), } init_config(build_options=build_options) ec = EasyConfig(toy_ec) self.assertEqual(ec.dependencies()[2]['short_mod_name'], 'foobar/1.2.3') self.assertEqual(ec.dependencies()[2]['external_module'], True) metadata = { 'name': ['foo', 'bar'], 'version': ['1.2.3', '3.2.1'], 'prefix': '/foo/bar', } self.assertEqual(ec.dependencies()[2]['external_module_metadata'], metadata)
def test_dependency(self): """ test all possible ways of specifying dependencies """ self.contents = '\n'.join([ 'name = "pi"', 'version = "3.14"', 'homepage = "http://example.com"', 'description = "test easyconfig"', 'toolchain = {"name":"GCC", "version": "4.6.3"}', 'dependencies = [("first", "1.1"), {"name": "second", "version": "2.2"}]', 'builddependencies = [("first", "1.1"), {"name": "second", "version": "2.2"}]', ]) self.prep() eb = EasyConfig(self.eb_file, build_options={'valid_stops': self.all_stops}) # should include builddependencies self.assertEqual(len(eb.dependencies()), 4) self.assertEqual(len(eb.builddependencies()), 2) first = eb.dependencies()[0] second = eb.dependencies()[1] self.assertEqual(first['name'], "first") self.assertEqual(second['name'], "second") self.assertEqual(first['version'], "1.1") self.assertEqual(second['version'], "2.2") self.assertEqual(det_full_ec_version(first), '1.1-GCC-4.6.3') self.assertEqual(det_full_ec_version(second), '2.2-GCC-4.6.3') # same tests for builddependencies first = eb.builddependencies()[0] second = eb.builddependencies()[1] self.assertEqual(first['name'], "first") self.assertEqual(second['name'], "second") self.assertEqual(first['version'], "1.1") self.assertEqual(second['version'], "2.2") self.assertEqual(det_full_ec_version(first), '1.1-GCC-4.6.3') self.assertEqual(det_full_ec_version(second), '2.2-GCC-4.6.3') self.assertErrorRegex(EasyBuildError, "Dependency foo of unsupported type", eb._parse_dependency, "foo") self.assertErrorRegex(EasyBuildError, "without name", eb._parse_dependency, ()) self.assertErrorRegex(EasyBuildError, "without version", eb._parse_dependency, {'name': 'test'})
def runTest(self): """ test all possible ways of specifying dependencies """ eb = EasyConfig(self.eb_file, valid_stops=self.all_stops) # should include builddependencies self.assertEqual(len(eb.dependencies()), 4) self.assertEqual(len(eb.builddependencies()), 2) first = eb.dependencies()[0] second = eb.dependencies()[1] self.assertEqual(first['name'], "first") self.assertEqual(second['name'], "second") self.assertEqual(first['version'], "1.1") self.assertEqual(second['version'], "2.2") self.assertEqual(first['tc'], '1.1-GCC-4.6.3') self.assertEqual(second['tc'], '2.2-GCC-4.6.3') # same tests for builddependencies first = eb.builddependencies()[0] second = eb.builddependencies()[1] self.assertEqual(first['name'], "first") self.assertEqual(second['name'], "second") self.assertEqual(first['version'], "1.1") self.assertEqual(second['version'], "2.2") self.assertEqual(first['tc'], '1.1-GCC-4.6.3') self.assertEqual(second['tc'], '2.2-GCC-4.6.3') eb['dependencies'] = ["wrong type"] self.assertErrorRegex(EasyBuildError, "wrong type from unsupported type", eb.dependencies) eb['dependencies'] = [()] self.assertErrorRegex(EasyBuildError, "without name", eb.dependencies) eb['dependencies'] = [{'name': "test"}] self.assertErrorRegex(EasyBuildError, "without version", eb.dependencies)
def test_update(self): """Test use of update() method for EasyConfig instances.""" toy_ebfile = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'easyconfigs', 'toy-0.0.eb') ec = EasyConfig(toy_ebfile) # for string values: append ec.update('unpack_options', '--strip-components=1') self.assertEqual(ec['unpack_options'].strip(), '--strip-components=1') ec.update('description', "- just a test") self.assertEqual(ec['description'].strip(), "Toy C program. - just a test") # spaces in between multiple updates for stirng values ec.update('configopts', 'CC="$CC"') ec.update('configopts', 'CXX="$CXX"') self.assertTrue(ec['configopts'].strip().endswith('CC="$CC" CXX="$CXX"')) # for list values: extend ec.update('patches', ['foo.patch', 'bar.patch']) self.assertEqual(ec['patches'], ['toy-0.0_typo.patch', 'foo.patch', 'bar.patch'])
def test_tweaking(self): """test tweaking ability of easyconfigs""" fd, tweaked_fn = tempfile.mkstemp(prefix='easybuild-tweaked-', suffix='.eb') os.close(fd) patches = ["t1.patch", ("t2.patch", 1), ("t3.patch", "test"), ("t4.h", "include")] self.contents = '\n'.join([ 'name = "pi"', 'homepage = "http://www.google.com"', 'description = "dummy description"', 'version = "3.14"', 'toolchain = {"name":"GCC", "version": "4.6.3"}', 'patches = %s', ]) % str(patches) self.prep() ver = "1.2.3" verpref = "myprefix" versuff = "mysuffix" tcname = "mytc" tcver = "4.1.2" new_patches = ['t5.patch', 't6.patch'] homepage = "http://www.justatest.com" tweaks = { 'version': ver, 'versionprefix': verpref, 'versionsuffix': versuff, 'toolchain_version': tcver, 'patches': new_patches } tweak(self.eb_file, tweaked_fn, tweaks) eb = EasyConfig(tweaked_fn, valid_stops=self.all_stops) self.assertEqual(eb['version'], ver) self.assertEqual(eb['versionprefix'], verpref) self.assertEqual(eb['versionsuffix'], versuff) self.assertEqual(eb['toolchain']['version'], tcver) self.assertEqual(eb['patches'], new_patches) eb = EasyConfig(self.eb_file, valid_stops=self.all_stops) # eb['toolchain']['version'] = tcver does not work as expected with templating enabled eb.enable_templating = False eb['version'] = ver eb['toolchain']['version'] = tcver eb.enable_templating = True eb.dump(self.eb_file) tweaks = { 'toolchain_name': tcname, 'patches': new_patches[:1], 'homepage': homepage, 'foo': "bar" } tweak(self.eb_file, tweaked_fn, tweaks) eb = EasyConfig(tweaked_fn, valid_stops=self.all_stops) self.assertEqual(eb['toolchain']['name'], tcname) self.assertEqual(eb['toolchain']['version'], tcver) self.assertEqual(eb['patches'], new_patches[:1]) self.assertEqual(eb['version'], ver) self.assertEqual(eb['homepage'], homepage) # specify patches as string, eb should promote it to a list because original value was a list tweaks['patches'] = new_patches[0] eb = EasyConfig(tweaked_fn, valid_stops=self.all_stops) self.assertEqual(eb['patches'], [new_patches[0]]) # cleanup os.remove(tweaked_fn)
def test_obtain_easyconfig(self): """test obtaining an easyconfig file given certain specifications""" tcname = 'GCC' tcver = '4.3.2' patches = ["one.patch"] # prepare a couple of eb files to test again fns = ["pi-3.14.eb", "pi-3.13-GCC-4.3.2.eb", "pi-3.15-GCC-4.3.2.eb", "pi-3.15-GCC-4.4.5.eb", "foo-1.2.3-GCC-4.3.2.eb"] eb_files = [(fns[0], "\n".join([ 'easyblock = "ConfigureMake"', 'name = "pi"', 'version = "3.12"', 'homepage = "http://example.com"', 'description = "test easyconfig"', 'toolchain = {"name": "dummy", "version": "dummy"}', 'patches = %s' % patches ])), (fns[1], "\n".join([ 'easyblock = "ConfigureMake"', 'name = "pi"', 'version = "3.13"', 'homepage = "http://example.com"', 'description = "test easyconfig"', 'toolchain = {"name": "%s", "version": "%s"}' % (tcname, tcver), 'patches = %s' % patches ])), (fns[2], "\n".join([ 'easyblock = "ConfigureMake"', 'name = "pi"', 'version = "3.15"', 'homepage = "http://example.com"', 'description = "test easyconfig"', 'toolchain = {"name": "%s", "version": "%s"}' % (tcname, tcver), 'patches = %s' % patches ])), (fns[3], "\n".join([ 'easyblock = "ConfigureMake"', 'name = "pi"', 'version = "3.15"', 'homepage = "http://example.com"', 'description = "test easyconfig"', 'toolchain = {"name": "%s", "version": "4.5.1"}' % tcname, 'patches = %s' % patches ])), (fns[4], "\n".join([ 'easyblock = "ConfigureMake"', 'name = "foo"', 'version = "1.2.3"', 'homepage = "http://example.com"', 'description = "test easyconfig"', 'toolchain = {"name": "%s", "version": "%s"}' % (tcname, tcver), 'foo_extra1 = "bar"', ])) ] self.ec_dir = tempfile.mkdtemp() for (fn, txt) in eb_files: write_file(os.path.join(self.ec_dir, fn), txt) # should crash when no suited easyconfig file (or template) is available specs = {'name': 'nosuchsoftware'} error_regexp = ".*No easyconfig files found for software %s, and no templates available. I'm all out of ideas." % specs['name'] self.assertErrorRegex(EasyBuildError, error_regexp, obtain_ec_for, specs, [self.ec_dir], None) # should find matching easyconfig file specs = { 'name': 'foo', 'version': '1.2.3' } res = obtain_ec_for(specs, [self.ec_dir], None) self.assertEqual(res[0], False) self.assertEqual(res[1], os.path.join(self.ec_dir, fns[-1])) # should not pick between multiple available toolchain names name = "pi" ver = "3.12" suff = "mysuff" specs.update({ 'name': name, 'version': ver, 'versionsuffix': suff }) error_regexp = ".*No toolchain name specified, and more than one available: .*" self.assertErrorRegex(EasyBuildError, error_regexp, obtain_ec_for, specs, [self.ec_dir], None) # should be able to generate an easyconfig file that slightly differs ver = '3.16' specs.update({ 'toolchain_name': tcname, 'toolchain_version': tcver, 'version': ver, 'start_dir': 'bar123' }) res = obtain_ec_for(specs, [self.ec_dir], None) self.assertEqual(res[1], "%s-%s-%s-%s%s.eb" % (name, ver, tcname, tcver, suff)) self.assertEqual(res[0], True) ec = EasyConfig(res[1]) self.assertEqual(ec['name'], specs['name']) self.assertEqual(ec['version'], specs['version']) self.assertEqual(ec['versionsuffix'], specs['versionsuffix']) self.assertEqual(ec['toolchain'], {'name': tcname, 'version': tcver}) self.assertEqual(ec['start_dir'], specs['start_dir']) os.remove(res[1]) specs.update({ 'foo': 'bar123' }) self.assertErrorRegex(EasyBuildError, "Unkown easyconfig parameter: foo", obtain_ec_for, specs, [self.ec_dir], None) del specs['foo'] # should pick correct version, i.e. not newer than what's specified, if a choice needs to be made ver = '3.14' specs.update({'version': ver}) res = obtain_ec_for(specs, [self.ec_dir], None) self.assertEqual(res[0], True) ec = EasyConfig(res[1]) self.assertEqual(ec['version'], specs['version']) txt = read_file(res[1]) self.assertTrue(re.search("^version = [\"']%s[\"']$" % ver, txt, re.M)) os.remove(res[1]) # should pick correct toolchain version as well, i.e. now newer than what's specified, if a choice needs to be made specs.update({ 'version': '3.15', 'toolchain_version': '4.4.5', }) res = obtain_ec_for(specs, [self.ec_dir], None) self.assertEqual(res[0], True) ec = EasyConfig(res[1]) self.assertEqual(ec['version'], specs['version']) self.assertEqual(ec['toolchain']['version'], specs['toolchain_version']) txt = read_file(res[1]) pattern = "^toolchain = .*version.*[\"']%s[\"'].*}$" % specs['toolchain_version'] self.assertTrue(re.search(pattern, txt, re.M)) os.remove(res[1]) # should be able to prepend to list of patches and handle list of dependencies new_patches = ['two.patch', 'three.patch'] specs.update({ 'patches': new_patches[:], 'dependencies': [('foo', '1.2.3'), ('bar', '666', '-bleh', ('gompi', '1.4.10'))], 'hiddendependencies': [('test', '3.2.1')], }) parsed_deps = [ { 'name': 'foo', 'version': '1.2.3', 'versionsuffix': '', 'toolchain': ec['toolchain'], 'dummy': False, 'short_mod_name': 'foo/1.2.3-GCC-4.4.5', 'full_mod_name': 'foo/1.2.3-GCC-4.4.5', 'hidden': False, 'external_module': False, 'external_module_metadata': {}, }, { 'name': 'bar', 'version': '666', 'versionsuffix': '-bleh', 'toolchain': {'name': 'gompi', 'version': '1.4.10'}, 'dummy': False, 'short_mod_name': 'bar/666-gompi-1.4.10-bleh', 'full_mod_name': 'bar/666-gompi-1.4.10-bleh', 'hidden': False, 'external_module': False, 'external_module_metadata': {}, }, { 'name': 'test', 'version': '3.2.1', 'versionsuffix': '', 'toolchain': ec['toolchain'], 'dummy': False, 'short_mod_name': 'test/.3.2.1-GCC-4.4.5', 'full_mod_name': 'test/.3.2.1-GCC-4.4.5', 'hidden': True, 'external_module': False, 'external_module_metadata': {}, }, ] # hidden dependencies must be included in list of dependencies res = obtain_ec_for(specs, [self.ec_dir], None) self.assertEqual(res[0], True) error_pattern = "Hidden dependencies with visible module names .* not in list of dependencies: .*" self.assertErrorRegex(EasyBuildError, error_pattern, EasyConfig, res[1]) specs['dependencies'].append(('test', '3.2.1')) res = obtain_ec_for(specs, [self.ec_dir], None) self.assertEqual(res[0], True) ec = EasyConfig(res[1]) self.assertEqual(ec['patches'], specs['patches']) self.assertEqual(ec.dependencies(), parsed_deps) # hidden dependencies are filtered from list of dependencies self.assertFalse('test/3.2.1-GCC-4.4.5' in [d['full_mod_name'] for d in ec['dependencies']]) self.assertTrue('test/.3.2.1-GCC-4.4.5' in [d['full_mod_name'] for d in ec['hiddendependencies']]) os.remove(res[1]) # hidden dependencies are also filtered from list of dependencies when validation is skipped res = obtain_ec_for(specs, [self.ec_dir], None) ec = EasyConfig(res[1], validate=False) self.assertFalse('test/3.2.1-GCC-4.4.5' in [d['full_mod_name'] for d in ec['dependencies']]) self.assertTrue('test/.3.2.1-GCC-4.4.5' in [d['full_mod_name'] for d in ec['hiddendependencies']]) os.remove(res[1]) # verify append functionality for lists specs['patches'].insert(0, '') res = obtain_ec_for(specs, [self.ec_dir], None) self.assertEqual(res[0], True) ec = EasyConfig(res[1]) self.assertEqual(ec['patches'], patches + new_patches) specs['patches'].remove('') os.remove(res[1]) # verify prepend functionality for lists specs['patches'].append('') res = obtain_ec_for(specs, [self.ec_dir], None) self.assertEqual(res[0], True) ec = EasyConfig(res[1]) self.assertEqual(ec['patches'], new_patches + patches) os.remove(res[1]) # should use supplied filename fn = "my.eb" res = obtain_ec_for(specs, [self.ec_dir], fn) self.assertEqual(res[0], True) self.assertEqual(res[1], fn) os.remove(res[1]) # should use a template if it's there tpl_path = os.path.join("share", "easybuild", "easyconfigs", "TEMPLATE.eb") def trim_path(path): dirs = path.split(os.path.sep) if len(dirs) > 3 and 'site-packages' in dirs: if path.endswith('.egg'): path = os.path.join(*dirs[:-4]) # strip of lib/python2.7/site-packages/*.egg part else: path = os.path.join(*dirs[:-3]) # strip of lib/python2.7/site-packages part return path tpl_full_path = find_full_path(tpl_path, trim=trim_path) # only run this test if the TEMPLATE.eb file is available # TODO: use unittest.skip for this (but only works from Python 2.7) if tpl_full_path: shutil.copy2(tpl_full_path, self.ec_dir) specs.update({'name': 'nosuchsoftware'}) res = obtain_ec_for(specs, [self.ec_dir], None) self.assertEqual(res[0], True) ec = EasyConfig(res[1]) self.assertEqual(ec['name'], specs['name']) os.remove(res[1]) # cleanup shutil.rmtree(self.ec_dir)
def test_robot_find_minimal_toolchain_of_dependency(self): """Test robot_find_minimal_toolchain_of_dependency.""" # replace log.experimental with log.warning to allow experimental code easybuild.framework.easyconfig.tools._log.experimental = easybuild.framework.easyconfig.tools._log.warning test_easyconfigs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') init_config(build_options={ 'valid_module_classes': module_classes(), 'robot_path': test_easyconfigs, }) # # First test that it can do basic resolution # gzip15 = { 'name': 'gzip', 'version': '1.5', 'versionsuffix': '', 'toolchain': {'name': 'goolf', 'version': '1.4.10'}, } get_toolchain_hierarchy.clear() new_gzip15_toolchain = robot_find_minimal_toolchain_of_dependency(gzip15, self.modtool) self.assertEqual(new_gzip15_toolchain, gzip15['toolchain']) # no easyconfig for gzip 1.4 with matching non-dummy (sub)toolchain gzip14 = { 'name': 'gzip', 'version': '1.4', 'versionsuffix': '', 'toolchain': {'name': 'goolf', 'version': '1.4.10'}, } get_toolchain_hierarchy.clear() self.assertEqual(robot_find_minimal_toolchain_of_dependency(gzip14, self.modtool), None) gzip14['toolchain'] = {'name': 'gompi', 'version': '1.4.10'} # # Second test also including dummy toolchain # init_config(build_options={ 'add_dummy_to_minimal_toolchains': True, 'valid_module_classes': module_classes(), 'robot_path': test_easyconfigs, }) # specify alternative parent toolchain gompi_1410 = {'name': 'gompi', 'version': '1.4.10'} get_toolchain_hierarchy.clear() new_gzip14_toolchain = robot_find_minimal_toolchain_of_dependency(gzip14, self.modtool, parent_tc=gompi_1410) self.assertTrue(new_gzip14_toolchain != gzip14['toolchain']) self.assertEqual(new_gzip14_toolchain, {'name': 'dummy', 'version': ''}) # default: use toolchain from dependency gzip14['toolchain'] = gompi_1410 get_toolchain_hierarchy.clear() new_gzip14_toolchain = robot_find_minimal_toolchain_of_dependency(gzip14, self.modtool) self.assertTrue(new_gzip14_toolchain != gzip14['toolchain']) self.assertEqual(new_gzip14_toolchain, {'name': 'dummy', 'version': ''}) # check reversed order (parent tc first) and skipping of parent tc itself dep = { 'name': 'SQLite', 'version': '3.8.10.2', 'toolchain': {'name': 'goolf', 'version': '1.4.10'}, } res = robot_find_minimal_toolchain_of_dependency(dep, self.modtool) self.assertEqual(res, {'name': 'GCC', 'version': '4.7.2'}) res = robot_find_minimal_toolchain_of_dependency(dep, self.modtool, parent_first=True) self.assertEqual(res, {'name': 'goolf', 'version': '1.4.10'}) # # Finally test if it can recognise existing modules and use those # barec = os.path.join(self.test_prefix, 'bar-1.2.3-goolf-1.4.10.eb') barec_txt = '\n'.join([ "easyblock = 'ConfigureMake'", "name = 'bar'", "version = '1.2.3'", "homepage = 'http://example.com'", "description = 'foo'", "toolchain = {'name': 'goolf', 'version': '1.4.10'}", # deliberately listing components of toolchain as dependencies without specifying subtoolchains, # to test resolving of dependencies with minimal toolchain # for each of these, we know test easyconfigs are available (which are required here) "dependencies = [", " ('OpenMPI', '1.6.4'),", # available with GCC/4.7.2 " ('OpenBLAS', '0.2.6', '-LAPACK-3.4.2'),", # available with gompi/1.4.10 " ('ScaLAPACK', '2.0.2', '-OpenBLAS-0.2.6-LAPACK-3.4.2'),", # available with gompi/1.4.10 " ('SQLite', '3.8.10.2'),", # available with goolf/1.4.10, gompi/1.4.10 and GCC/4.7.2 "]", ]) write_file(barec, barec_txt) # check without --minimal-toolchains init_config(build_options={ 'valid_module_classes': module_classes(), 'robot_path': test_easyconfigs, }) bar = EasyConfig(barec) expected_dep_versions = [ '1.6.4-GCC-4.7.2', '0.2.6-gompi-1.4.10-LAPACK-3.4.2', '2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2', '3.8.10.2-goolf-1.4.10', ] for dep, expected_dep_version in zip(bar.dependencies(), expected_dep_versions): self.assertEqual(det_full_ec_version(dep), expected_dep_version) # check with --minimal-toolchains enabled init_config(build_options={ 'minimal_toolchains': True, 'valid_module_classes': module_classes(), 'robot_path': test_easyconfigs, }) bar = EasyConfig(barec) # check that all bar dependencies have been processed as expected expected_dep_versions[-1] = '3.8.10.2-GCC-4.7.2' for dep, expected_dep_version in zip(bar.dependencies(), expected_dep_versions): self.assertEqual(det_full_ec_version(dep), expected_dep_version) # Add the gompi/1.4.10 version of SQLite as an available module module_parent = os.path.join(self.test_prefix, 'minimal_toolchain_modules') module_file = os.path.join(module_parent, 'SQLite', '3.8.10.2-gompi-1.4.10') module_txt = '\n'.join([ "#%Module", "set root /tmp/SQLite/3.8.10.2", "setenv EBROOTSQLITE $root", "setenv EBVERSIONSQLITE 3.8.10.2", "setenv EBDEVELSQLITE $root/easybuild/SQLite-3.8.10.2-easybuild-devel", ]) write_file(module_file, module_txt) os.environ['MODULEPATH'] = module_parent # Add the parent directory to the MODULEPATH invalidate_module_caches_for(module_parent) # Reinitialize the environment for the updated MODULEPATH and use_existing_modules init_config(build_options={ 'minimal_toolchains': True, 'use_existing_modules': True, 'valid_module_classes': module_classes(), 'robot_path': test_easyconfigs, }) # Check gompi is now being picked up bar = EasyConfig(barec) # Re-parse the parent easyconfig sqlite = bar.dependencies()[3] self.assertEqual(det_full_ec_version(sqlite), '3.8.10.2-gompi-1.4.10') # Add the goolf version as an available version and check that gets precedence over the gompi version module_file = os.path.join(module_parent, 'SQLite', '3.8.10.2-goolf-1.4.10') write_file(module_file, module_txt) invalidate_module_caches_for(module_parent) bar = EasyConfig(barec) # Re-parse the parent easyconfig sqlite = bar.dependencies()[3] self.assertEqual(det_full_ec_version(sqlite), '3.8.10.2-goolf-1.4.10')
# fs.walk yields the same results as os.walk, so should be interchangable # same for fs.join and os.path.join for root, subfolders, files in walk(options.path): if '.git' in subfolders: log.info("found .git subfolder, ignoring it") subfolders.remove('.git') for ec_file in files: if not ec_file.endswith('.eb') or ec_file in ["TEMPLATE.eb"]: log.warning("SKIPPING %s/%s" % (root, ec_file)) continue ec_file = join(root, ec_file) ec_file = read(ec_file) try: ec = EasyConfig(ec_file) log.info("found valid easyconfig %s" % ec) if not ec.name in names: log.info("found new software package %s" % ec) # check if an easyblock exists module = get_easyblock_class(None, name=ec.name).__module__.split('.')[-1] if module != "configuremake": ec.easyblock = module else: ec.easyblock = None configs.append(ec) names.append(ec.name) except Exception, err: log.error("faulty easyconfig %s: %s" % (ec_file, err)) log.info("Found easyconfigs: %s" % [x.name for x in configs])